Spaces:
Paused
Paused
yaGeey commited on
Commit ·
94f96fe
1
Parent(s): 9a69639
create and close browser for each action to clear ram
Browse files- src/browser.ts +27 -27
- src/hashHandlers.ts +26 -14
- src/index.ts +14 -14
src/browser.ts
CHANGED
|
@@ -1,27 +1,29 @@
|
|
| 1 |
import type { Browser, BrowserContext, Page } from 'playwright'
|
| 2 |
import { chromium } from 'playwright-extra'
|
|
|
|
| 3 |
|
| 4 |
-
|
| 5 |
-
export let glContext: BrowserContext | null = null
|
| 6 |
-
export let glPage: Page | null = null
|
| 7 |
|
| 8 |
-
export async function
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
// launch browser
|
| 12 |
-
if (glBrowser) await glBrowser.close() // if not connected - close and create new one
|
| 13 |
-
glBrowser = await chromium.launch({
|
| 14 |
headless: true,
|
| 15 |
-
args: [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
})
|
| 17 |
-
|
| 18 |
-
// create context
|
| 19 |
-
glContext = await glBrowser.newContext({
|
| 20 |
locale: 'en-US',
|
| 21 |
timezoneId: 'America/New_York',
|
| 22 |
bypassCSP: true,
|
| 23 |
})
|
| 24 |
-
await
|
| 25 |
{
|
| 26 |
name: 'sp_dc',
|
| 27 |
value: process.env.SP_DC!,
|
|
@@ -41,21 +43,19 @@ export async function ensureBrowser() {
|
|
| 41 |
sameSite: 'None',
|
| 42 |
},
|
| 43 |
])
|
| 44 |
-
|
| 45 |
-
/
|
| 46 |
-
|
| 47 |
-
await glPage.route('**/*.{png,jpg,jpeg,gif,woff,woff2,sentry}', (r) => r.abort())
|
| 48 |
}
|
| 49 |
|
| 50 |
-
export async function
|
| 51 |
-
// close and clear browser
|
| 52 |
-
if (glBrowser) {
|
| 53 |
-
await glBrowser.close().catch(() => {})
|
| 54 |
-
glBrowser = null
|
| 55 |
-
}
|
| 56 |
-
|
| 57 |
-
// handle error
|
| 58 |
const details = error instanceof Error ? error.message : 'Unknown error'
|
| 59 |
-
console.error('Error:', details)
|
| 60 |
return details
|
| 61 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import type { Browser, BrowserContext, Page } from 'playwright'
|
| 2 |
import { chromium } from 'playwright-extra'
|
| 3 |
+
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
|
| 4 |
|
| 5 |
+
chromium.use(StealthPlugin())
|
|
|
|
|
|
|
| 6 |
|
| 7 |
+
export async function createInstance() {
|
| 8 |
+
console.log(`-> Launching browser`)
|
| 9 |
+
const browser = await chromium.launch({
|
|
|
|
|
|
|
|
|
|
| 10 |
headless: true,
|
| 11 |
+
args: [
|
| 12 |
+
'--disable-dev-shm-usage',
|
| 13 |
+
'--no-sandbox',
|
| 14 |
+
'--disable-setuid-sandbox',
|
| 15 |
+
'--disable-gpu',
|
| 16 |
+
'--no-first-run',
|
| 17 |
+
'--single-process',
|
| 18 |
+
'--mute-audio',
|
| 19 |
+
],
|
| 20 |
})
|
| 21 |
+
const context = await browser.newContext({
|
|
|
|
|
|
|
| 22 |
locale: 'en-US',
|
| 23 |
timezoneId: 'America/New_York',
|
| 24 |
bypassCSP: true,
|
| 25 |
})
|
| 26 |
+
await context.addCookies([
|
| 27 |
{
|
| 28 |
name: 'sp_dc',
|
| 29 |
value: process.env.SP_DC!,
|
|
|
|
| 43 |
sameSite: 'None',
|
| 44 |
},
|
| 45 |
])
|
| 46 |
+
const page = await context.newPage()
|
| 47 |
+
await page.route('**/*.{png,jpg,jpeg,gif,woff,woff2,sentry}', (r) => r.abort())
|
| 48 |
+
return { browser, context, page }
|
|
|
|
| 49 |
}
|
| 50 |
|
| 51 |
+
export async function handleError(error: unknown) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
const details = error instanceof Error ? error.message : 'Unknown error'
|
| 53 |
+
console.error('💥 Error:', details)
|
| 54 |
return details
|
| 55 |
}
|
| 56 |
+
|
| 57 |
+
export async function killBrowser(browser: Browser | null) {
|
| 58 |
+
console.log(`<- Closing browser`)
|
| 59 |
+
if (browser) await browser.close().catch(() => {})
|
| 60 |
+
if (global.gc) global.gc() // force garbage collection to free RAM
|
| 61 |
+
}
|
src/hashHandlers.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
import { addToPlaylistAction } from './actions.js'
|
| 2 |
-
import {
|
| 3 |
import { type Operation, store } from './storage.js'
|
| 4 |
import { delay } from './utils.js'
|
| 5 |
|
|
@@ -17,37 +17,49 @@ export const operations: Operation[] = [
|
|
| 17 |
]
|
| 18 |
|
| 19 |
export async function updateHash(op: Operation) {
|
| 20 |
-
|
| 21 |
-
await ensureBrowser()
|
| 22 |
-
const page = glPage!
|
| 23 |
|
| 24 |
-
|
| 25 |
const hashPromise = page
|
| 26 |
.waitForResponse(
|
| 27 |
async (res) => {
|
| 28 |
const url = res.url()
|
| 29 |
if (url.includes('query') || (url.includes('graphql') && res.status() === 200)) {
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
}
|
| 36 |
return false
|
| 37 |
},
|
| 38 |
{ timeout: 60000 },
|
| 39 |
)
|
| 40 |
.then((res) => (res.request().postDataJSON()?.extensions?.persistedQuery?.sha256Hash || null) as string | null)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
-
// load the page
|
| 43 |
await page.goto(op.url, { waitUntil: 'domcontentloaded', timeout: 60000 })
|
| 44 |
|
| 45 |
-
|
| 46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
return hash
|
| 48 |
} catch (err) {
|
| 49 |
-
|
| 50 |
return null
|
|
|
|
|
|
|
| 51 |
}
|
| 52 |
}
|
| 53 |
|
|
|
|
| 1 |
import { addToPlaylistAction } from './actions.js'
|
| 2 |
+
import { createInstance, killBrowser, handleError } from './browser.js'
|
| 3 |
import { type Operation, store } from './storage.js'
|
| 4 |
import { delay } from './utils.js'
|
| 5 |
|
|
|
|
| 17 |
]
|
| 18 |
|
| 19 |
export async function updateHash(op: Operation) {
|
| 20 |
+
const { browser, page } = await createInstance()
|
|
|
|
|
|
|
| 21 |
|
| 22 |
+
try {
|
| 23 |
const hashPromise = page
|
| 24 |
.waitForResponse(
|
| 25 |
async (res) => {
|
| 26 |
const url = res.url()
|
| 27 |
if (url.includes('query') || (url.includes('graphql') && res.status() === 200)) {
|
| 28 |
+
try {
|
| 29 |
+
const body = res.request().postDataJSON()
|
| 30 |
+
if (!body) return false
|
| 31 |
+
const hash = body.extensions?.persistedQuery?.sha256Hash
|
| 32 |
+
if (hash) store.hashes[body.operationName] = hash
|
| 33 |
+
return body.operationName === op.name
|
| 34 |
+
} catch {
|
| 35 |
+
return false
|
| 36 |
+
}
|
| 37 |
}
|
| 38 |
return false
|
| 39 |
},
|
| 40 |
{ timeout: 60000 },
|
| 41 |
)
|
| 42 |
.then((res) => (res.request().postDataJSON()?.extensions?.persistedQuery?.sha256Hash || null) as string | null)
|
| 43 |
+
.catch((err) => {
|
| 44 |
+
console.warn(`⚠️ [${op.name}] Hash listener ended: ${err.message}`)
|
| 45 |
+
return null
|
| 46 |
+
})
|
| 47 |
|
|
|
|
| 48 |
await page.goto(op.url, { waitUntil: 'domcontentloaded', timeout: 60000 })
|
| 49 |
|
| 50 |
+
const actionPromise = op.action
|
| 51 |
+
? op.action(page).catch((e) => {
|
| 52 |
+
throw new Error(`Action failed: ${e.message}`)
|
| 53 |
+
})
|
| 54 |
+
: Promise.resolve()
|
| 55 |
+
|
| 56 |
+
const [hash] = await Promise.all([hashPromise, actionPromise])
|
| 57 |
return hash
|
| 58 |
} catch (err) {
|
| 59 |
+
handleError(err)
|
| 60 |
return null
|
| 61 |
+
} finally {
|
| 62 |
+
await killBrowser(browser)
|
| 63 |
}
|
| 64 |
}
|
| 65 |
|
src/index.ts
CHANGED
|
@@ -1,17 +1,15 @@
|
|
| 1 |
import express from 'express'
|
| 2 |
-
import { chromium } from 'playwright-extra'
|
| 3 |
import 'dotenv/config'
|
| 4 |
-
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
|
| 5 |
import PQueue from 'p-queue'
|
| 6 |
import cron from 'node-cron'
|
| 7 |
import { store, type AccessTokenResponse, type TokenResponse } from './storage.js'
|
| 8 |
-
import {
|
| 9 |
import { delay } from './utils.js'
|
| 10 |
import { updateAllHashes, operations, updateHash } from './hashHandlers.js'
|
|
|
|
| 11 |
|
| 12 |
const app = express()
|
| 13 |
const PORT = process.env.PORT || 3000
|
| 14 |
-
chromium.use(StealthPlugin())
|
| 15 |
const queue = new PQueue({ concurrency: 1 })
|
| 16 |
|
| 17 |
// Health check - must be before auth middleware
|
|
@@ -44,6 +42,7 @@ function isTokenValid(): boolean {
|
|
| 44 |
}
|
| 45 |
|
| 46 |
app.get('/token', async (req, res) => {
|
|
|
|
| 47 |
try {
|
| 48 |
// return token if valid
|
| 49 |
if (isTokenValid()) {
|
|
@@ -58,8 +57,8 @@ app.get('/token', async (req, res) => {
|
|
| 58 |
return { access: store.access!, client: store.client! } satisfies TokenResponse
|
| 59 |
}
|
| 60 |
|
| 61 |
-
await
|
| 62 |
-
|
| 63 |
|
| 64 |
// access token
|
| 65 |
const accessTokenPromise = page
|
|
@@ -99,9 +98,12 @@ app.get('/token', async (req, res) => {
|
|
| 99 |
}
|
| 100 |
return { access: store.access!, client: store.client! } satisfies TokenResponse
|
| 101 |
})
|
|
|
|
|
|
|
| 102 |
res.json(result)
|
| 103 |
-
} catch (
|
| 104 |
-
const details =
|
|
|
|
| 105 |
res.status(500).json({ error: 'Failed to get token', details })
|
| 106 |
}
|
| 107 |
})
|
|
@@ -138,12 +140,10 @@ app.put('/hashes', async (req, res) => {
|
|
| 138 |
return res.status(400).json({ error: 'No valid operation names provided' })
|
| 139 |
}
|
| 140 |
if (hashes.length !== names.length) {
|
| 141 |
-
return res
|
| 142 |
-
|
| 143 |
-
.
|
| 144 |
-
|
| 145 |
-
details: { raw, hashes: Object.fromEntries(hashes.map((i) => [i.name, i.hash])) },
|
| 146 |
-
})
|
| 147 |
}
|
| 148 |
}
|
| 149 |
|
|
|
|
| 1 |
import express from 'express'
|
|
|
|
| 2 |
import 'dotenv/config'
|
|
|
|
| 3 |
import PQueue from 'p-queue'
|
| 4 |
import cron from 'node-cron'
|
| 5 |
import { store, type AccessTokenResponse, type TokenResponse } from './storage.js'
|
| 6 |
+
import { createInstance, killBrowser, handleError } from './browser.js'
|
| 7 |
import { delay } from './utils.js'
|
| 8 |
import { updateAllHashes, operations, updateHash } from './hashHandlers.js'
|
| 9 |
+
import type { Browser } from 'playwright'
|
| 10 |
|
| 11 |
const app = express()
|
| 12 |
const PORT = process.env.PORT || 3000
|
|
|
|
| 13 |
const queue = new PQueue({ concurrency: 1 })
|
| 14 |
|
| 15 |
// Health check - must be before auth middleware
|
|
|
|
| 42 |
}
|
| 43 |
|
| 44 |
app.get('/token', async (req, res) => {
|
| 45 |
+
let exBrowser: Browser | null = null
|
| 46 |
try {
|
| 47 |
// return token if valid
|
| 48 |
if (isTokenValid()) {
|
|
|
|
| 57 |
return { access: store.access!, client: store.client! } satisfies TokenResponse
|
| 58 |
}
|
| 59 |
|
| 60 |
+
const { browser, page } = await createInstance()
|
| 61 |
+
exBrowser = browser
|
| 62 |
|
| 63 |
// access token
|
| 64 |
const accessTokenPromise = page
|
|
|
|
| 98 |
}
|
| 99 |
return { access: store.access!, client: store.client! } satisfies TokenResponse
|
| 100 |
})
|
| 101 |
+
|
| 102 |
+
killBrowser(exBrowser)
|
| 103 |
res.json(result)
|
| 104 |
+
} catch (err) {
|
| 105 |
+
const details = handleError(err)
|
| 106 |
+
killBrowser(exBrowser)
|
| 107 |
res.status(500).json({ error: 'Failed to get token', details })
|
| 108 |
}
|
| 109 |
})
|
|
|
|
| 140 |
return res.status(400).json({ error: 'No valid operation names provided' })
|
| 141 |
}
|
| 142 |
if (hashes.length !== names.length) {
|
| 143 |
+
return res.status(404).json({
|
| 144 |
+
error: 'Some operation names were invalid',
|
| 145 |
+
details: { raw, hashes: Object.fromEntries(hashes.map((i) => [i.name, i.hash])) },
|
| 146 |
+
})
|
|
|
|
|
|
|
| 147 |
}
|
| 148 |
}
|
| 149 |
|