From bd4d1f57aca8a782120aa5eaa27924cf1f86bca4 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sat, 14 Feb 2026 11:03:37 +0100 Subject: [PATCH 1/2] feat: remove --render-js / render_heavy_js option from all commands The render_heavy_js feature has been removed from the API. This removes the --render-js CLI flag and render_heavy_js schema field from smart-scraper, markdownify, scrape, and crawl commands, along with all documentation references. Co-Authored-By: Claude Opus 4.6 --- README.md | 10 ++++------ package.json | 2 +- skills/just-scrape/SKILL.md | 13 ++++--------- src/commands/crawl.ts | 2 -- src/commands/markdownify.ts | 2 -- src/commands/scrape.ts | 2 -- src/commands/smart-scraper.ts | 2 -- src/lib/schemas.ts | 4 ---- 8 files changed, 9 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 1d223c5..f829372 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Extract structured data from any URL using AI. [docs](https://docs.scrapegraphai ```bash just-scrape smart-scraper -p # Extract data with AI just-scrape smart-scraper -p --schema # Enforce output schema -just-scrape smart-scraper -p --scrolls # Infinite scroll (0-100) +just-scrape smart-scraper -p --scrolls # Infinite scroll (0-100) just-scrape smart-scraper -p --pages # Multi-page (1-100) just-scrape smart-scraper -p --stealth # Anti-bot bypass (+4 credits) just-scrape smart-scraper -p --cookies --headers @@ -125,7 +125,7 @@ just-scrape smart-scraper https://news.example.com -p "Get all article headlines # Scrape a JS-heavy SPA behind anti-bot protection just-scrape smart-scraper https://app.example.com/dashboard -p "Extract user stats" \ - --render-js --stealth + --stealth ``` ## Search Scraper @@ -164,7 +164,6 @@ Convert any webpage to clean markdown. [docs](https://docs.scrapegraphai.com/ser ```bash just-scrape markdownify # Convert to markdown -just-scrape markdownify --render-js # JS rendering (+1 credit) just-scrape markdownify --stealth # Anti-bot bypass (+4 credits) just-scrape markdownify --headers # Custom headers ``` @@ -176,7 +175,7 @@ just-scrape markdownify --headers # Custom headers just-scrape markdownify https://blog.example.com/my-article # Convert a JS-rendered page behind Cloudflare -just-scrape markdownify https://protected.example.com --render-js --stealth +just-scrape markdownify https://protected.example.com --stealth # Pipe markdown to a file just-scrape markdownify https://docs.example.com/api --json | jq -r '.result' > api-docs.md @@ -196,7 +195,7 @@ just-scrape crawl --no-extraction --max-pages # Markdown only (2 cr just-scrape crawl -p --schema # Enforce output schema just-scrape crawl -p --rules # Crawl rules (include_paths, same_domain) just-scrape crawl -p --no-sitemap # Skip sitemap discovery -just-scrape crawl -p --render-js --stealth # JS + anti-bot +just-scrape crawl -p --stealth # Anti-bot bypass ``` ### Examples @@ -242,7 +241,6 @@ Get raw HTML content from a URL. [docs](https://docs.scrapegraphai.com/services/ ```bash just-scrape scrape # Raw HTML -just-scrape scrape --render-js # JS rendering (+1 credit) just-scrape scrape --stealth # Anti-bot bypass (+4 credits) just-scrape scrape --branding # Extract branding (+2 credits) just-scrape scrape --country-code # Geo-targeting diff --git a/package.json b/package.json index 2e60ab5..26ccbb7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "just-scrape", - "version": "0.1.4", + "version": "0.1.5", "description": "ScrapeGraph AI CLI tool", "type": "module", "main": "dist/cli.mjs", diff --git a/skills/just-scrape/SKILL.md b/skills/just-scrape/SKILL.md index 457240f..18acea8 100644 --- a/skills/just-scrape/SKILL.md +++ b/skills/just-scrape/SKILL.md @@ -47,7 +47,6 @@ API key resolution order: `SGAI_API_KEY` env var → `.env` file → `~/.scrapeg All commands support `--json` for machine-readable output (suppresses banner, spinners, prompts). Scraping commands share these optional flags: -- `--render-js` — render JavaScript (+1 credit) - `--stealth` — bypass anti-bot detection (+4 credits) - `--headers ` — custom HTTP headers as JSON string - `--schema ` — enforce output JSON schema @@ -63,7 +62,6 @@ just-scrape smart-scraper -p just-scrape smart-scraper -p --schema just-scrape smart-scraper -p --scrolls # infinite scroll (0-100) just-scrape smart-scraper -p --pages # multi-page (1-100) -just-scrape smart-scraper -p --render-js # JS rendering (+1 credit) just-scrape smart-scraper -p --stealth # anti-bot (+4 credits) just-scrape smart-scraper -p --cookies --headers just-scrape smart-scraper -p --plain-text @@ -80,7 +78,7 @@ just-scrape smart-scraper https://news.example.com -p "Get headlines and dates" # JS-heavy SPA behind anti-bot just-scrape smart-scraper https://app.example.com/dashboard -p "Extract user stats" \ - --render-js --stealth + --stealth ``` ### Search Scraper @@ -113,14 +111,13 @@ Convert any webpage to clean markdown. ```bash just-scrape markdownify -just-scrape markdownify --render-js # +1 credit just-scrape markdownify --stealth # +4 credits just-scrape markdownify --headers ``` ```bash just-scrape markdownify https://blog.example.com/my-article -just-scrape markdownify https://protected.example.com --render-js --stealth +just-scrape markdownify https://protected.example.com --stealth just-scrape markdownify https://docs.example.com/api --json | jq -r '.result' > api-docs.md ``` @@ -136,7 +133,7 @@ just-scrape crawl --no-extraction --max-pages # markdown only (2 cre just-scrape crawl -p --schema just-scrape crawl -p --rules # include_paths, same_domain just-scrape crawl -p --no-sitemap -just-scrape crawl -p --render-js --stealth +just-scrape crawl -p --stealth ``` ```bash @@ -157,7 +154,6 @@ Get raw HTML content from a URL. ```bash just-scrape scrape -just-scrape scrape --render-js # +1 credit just-scrape scrape --stealth # +4 credits just-scrape scrape --branding # extract logos/colors/fonts (+2 credits) just-scrape scrape --country-code @@ -269,7 +265,7 @@ done ```bash # JS-heavy SPA behind Cloudflare -just-scrape smart-scraper https://protected.example.com -p "Extract data" --render-js --stealth +just-scrape smart-scraper https://protected.example.com -p "Extract data" --stealth # With custom cookies/headers just-scrape smart-scraper https://example.com -p "Extract data" \ @@ -280,7 +276,6 @@ just-scrape smart-scraper https://example.com -p "Extract data" \ | Feature | Extra Credits | |---|---| -| `--render-js` | +1 per page | | `--stealth` | +4 per request | | `--branding` (scrape only) | +2 | | `search-scraper` extraction | 10 per request | diff --git a/src/commands/crawl.ts b/src/commands/crawl.ts index 4b294c6..924ab0e 100644 --- a/src/commands/crawl.ts +++ b/src/commands/crawl.ts @@ -28,7 +28,6 @@ export default defineCommand({ schema: { type: "string", description: "Output JSON schema (as JSON string)" }, rules: { type: "string", description: "Crawl rules as JSON object string" }, "no-sitemap": { type: "boolean", description: "Disable sitemap-based URL discovery" }, - "render-js": { type: "boolean", description: "Enable heavy JS rendering (+1 credit/page)" }, stealth: { type: "boolean", description: "Bypass bot detection (+4 credits)" }, json: { type: "boolean", description: "Output raw JSON (pipeable)" }, }, @@ -46,7 +45,6 @@ export default defineCommand({ if (args.schema) params.schema = JSON.parse(args.schema); if (args.rules) params.rules = JSON.parse(args.rules); if (args["no-sitemap"]) params.sitemap = false; - if (args["render-js"]) params.render_heavy_js = true; if (args.stealth) params.stealth = true; out.start("Crawling"); diff --git a/src/commands/markdownify.ts b/src/commands/markdownify.ts index a7c0f7f..467cb08 100644 --- a/src/commands/markdownify.ts +++ b/src/commands/markdownify.ts @@ -14,7 +14,6 @@ export default defineCommand({ description: "Website URL to convert", required: true, }, - "render-js": { type: "boolean", description: "Enable heavy JS rendering (+1 credit)" }, stealth: { type: "boolean", description: "Bypass bot detection (+4 credits)" }, headers: { type: "string", description: "Custom headers as JSON object string" }, json: { type: "boolean", description: "Output raw JSON (pipeable)" }, @@ -28,7 +27,6 @@ export default defineCommand({ website_url: args.url, }; - if (args["render-js"]) params.render_heavy_js = true; if (args.stealth) params.stealth = true; if (args.headers) params.headers = JSON.parse(args.headers); diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 90ae718..8fa5394 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -14,7 +14,6 @@ export default defineCommand({ description: "Website URL to scrape", required: true, }, - "render-js": { type: "boolean", description: "Enable heavy JS rendering (+1 credit)" }, stealth: { type: "boolean", description: "Bypass bot detection (+4 credits)" }, branding: { type: "boolean", description: "Extract branding info (+2 credits)" }, "country-code": { type: "string", description: "ISO country code for geo-targeting" }, @@ -27,7 +26,6 @@ export default defineCommand({ const params: scrapegraphai.ScrapeParams = { website_url: args.url }; - if (args["render-js"]) params.render_heavy_js = true; if (args.stealth) params.stealth = true; if (args.branding) params.branding = true; if (args["country-code"]) params.country_code = args["country-code"]; diff --git a/src/commands/smart-scraper.ts b/src/commands/smart-scraper.ts index f754c14..a44d63f 100644 --- a/src/commands/smart-scraper.ts +++ b/src/commands/smart-scraper.ts @@ -23,7 +23,6 @@ export default defineCommand({ schema: { type: "string", description: "Output JSON schema (as JSON string)" }, scrolls: { type: "string", description: "Number of infinite scrolls (0-100)" }, pages: { type: "string", description: "Total pages to scrape (1-100)" }, - "render-js": { type: "boolean", description: "Enable heavy JS rendering (+1 credit)" }, stealth: { type: "boolean", description: "Bypass bot detection (+4 credits)" }, cookies: { type: "string", description: "Cookies as JSON object string" }, headers: { type: "string", description: "Custom headers as JSON object string" }, @@ -43,7 +42,6 @@ export default defineCommand({ if (args.schema) params.output_schema = JSON.parse(args.schema); if (args.scrolls) params.number_of_scrolls = Number(args.scrolls); if (args.pages) params.total_pages = Number(args.pages); - if (args["render-js"]) params.render_heavy_js = true; if (args.stealth) params.stealth = true; if (args.cookies) params.cookies = JSON.parse(args.cookies); if (args.headers) params.headers = JSON.parse(args.headers); diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts index 5054498..7d8c585 100644 --- a/src/lib/schemas.ts +++ b/src/lib/schemas.ts @@ -11,7 +11,6 @@ export const SmartScraperSchema = z.object({ output_schema: jsonObject.optional(), number_of_scrolls: z.number().int().min(0).max(100).optional(), total_pages: z.number().int().min(1).max(100).optional(), - render_heavy_js: z.boolean().optional(), stealth: z.boolean().optional(), cookies: jsonStringObject.optional(), headers: jsonStringObject.optional(), @@ -31,7 +30,6 @@ export const SearchScraperSchema = z.object({ export const MarkdownifySchema = z.object({ website_url: z.string().url(), - render_heavy_js: z.boolean().optional(), stealth: z.boolean().optional(), headers: jsonStringObject.optional(), webhook_url: z.string().url().optional(), @@ -46,7 +44,6 @@ export const CrawlSchema = z.object({ schema: jsonObject.optional(), rules: jsonObject.optional(), sitemap: z.boolean().optional(), - render_heavy_js: z.boolean().optional(), stealth: z.boolean().optional(), webhook_url: z.string().url().optional(), }); @@ -62,7 +59,6 @@ export const SitemapSchema = z.object({ export const ScrapeSchema = z.object({ website_url: z.string().url(), - render_heavy_js: z.boolean().optional(), stealth: z.boolean().optional(), branding: z.boolean().optional(), country_code: z.string().optional(), From 4dedd12e9928e8c6f2854285aed5b5707b27b8f5 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Sun, 15 Feb 2026 19:50:34 +0100 Subject: [PATCH 2/2] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 26ccbb7..e49bd03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "just-scrape", - "version": "0.1.5", + "version": "0.1.6", "description": "ScrapeGraph AI CLI tool", "type": "module", "main": "dist/cli.mjs",