feat(wait): add idle and almost idle network

Author: j-mendez

cli/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "spider-cloud-cli"
-version = "0.1.80"
+version = "0.1.82"
 edition = "2021"
 authors = [ "j-mendez <jeff@spider.cloud>"]
 description = "The Spider Cloud CLI for web crawling and scraping"

cli/Cargo.toml

@@ -1,6 +1,6 @@
 {
   "name": "@spider-cloud/spider-client",
-  "version": "0.1.80",
+  "version": "0.1.82",
   "description": "Isomorphic Javascript SDK for Spider Cloud services",
   "scripts": {
     "test": "node --import tsx  --test __tests__/*test.ts",

javascript/package.json

@@ -112,13 +112,17 @@ interface Delay {
 /**
  * Represents the wait_for configuration.
  * @typedef {Object} WaitFor
- * @property {IdleNetwork} [idle_network] - Configuration to wait for network to be idle.
+ * @property {IdleNetwork} [idle_network] - Configuration to wait for network to be idle between period.
+ * @property {IdleNetwork0} [idle_network] - Configuration to wait for network to be idle with max timeout.
+ * @property {AlmostIdleNetwork0} [idle_network] - Configuration to wait for network to almost idle with max timeout.
  * @property {Selector} [selector] - Configuration to wait for a CSS selector.
  * @property {Delay} [delay] - Configuration to wait for a delay.
  * @property {boolean} [page_navigations] - Whether to wait for page navigations.
  */
 export interface WaitForConfiguration {
   idle_network?: IdleNetwork;
+  idle_network0?: IdleNetwork;
+  almost_idle_network0?: IdleNetwork;
   selector?: Selector;
   dom?: Selector;
   delay?: Delay;
@@ -310,6 +314,32 @@ export type Proxy =
   | "mobile"
   | "isp";
 
+  export type LinkRewriteReplace = {
+  type: "replace";
+  /**
+   * Only apply when the link's host matches this value.
+   * Optional key; null means "no host filter".
+   */
+  host?: string | null;
+  find: string;
+  replace_with: string;
+};
+
+// Link rewrite regex.
+export type LinkRewriteRegex = {
+  type: "regex";
+  /**
+   * Only apply when the link's host matches this value.
+   * Optional key; null means "no host filter".
+   */
+  host?: string | null;
+  pattern: string;
+  replace_with: string;
+};
+
+// The link rewrite rule.
+export type LinkRewriteRule = LinkRewriteReplace | LinkRewriteRegex;
+
 /**
  * Represents the options available for making a spider request.
  */
@@ -424,6 +454,8 @@ export interface SpiderParams {
    */
   webhooks?: WebhookSettings;
 
+  link_rewrite?: LinkRewriteRule
+
   /**
    * Specifies whether to use fingerprinting protection.
    */

javascript/src/config.ts

@@ -8,7 +8,7 @@ def read_file(fname):
 
 setup(
     name="spider_client",
-    version="0.1.80",
+    version="0.1.82",
     url="https://github.com/spider-rs/spider-clients/tree/main/python",
     license="MIT",
     author="Spider",

python/setup.py

@@ -267,7 +267,7 @@ def _prepare_headers(
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": "AsyncSpider-Client/0.1.80",
+            "User-Agent": "AsyncSpider-Client/0.1.82",
         }
 
     async def _handle_error(self, response: ClientResponse, action: str) -> None:

python/spider/async_spider.py

@@ -280,7 +280,7 @@ def _prepare_headers(self, content_type: str = "application/json"):
         return {
             "Content-Type": content_type,
             "Authorization": f"Bearer {self.api_key}",
-            "User-Agent": f"Spider-Client/0.1.80",
+            "User-Agent": f"Spider-Client/0.1.82",
         }
 
     def _post_request(self, url: str, data, headers, stream=False):

python/spider/spider.py

@@ -170,6 +170,8 @@ class DelayDict(TypedDict):
 
 class WaitForDict(TypedDict, total=False):
     idle_network: Optional[IdleNetworkDict]
+    idle_network0: Optional[IdleNetworkDict]
+    almost_idle_network0: Optional[IdleNetworkDict]
     selector: Optional[SelectorDict]
     dom: Optional[SelectorDict]
     delay: Optional[DelayDict]
@@ -218,6 +220,23 @@ class Proxy(str, Enum):
     residential_core = "residential_core"            # Balanced core plan
     residential_plus = "residential_plus"            # Extended core pool
 
+class LinkRewriteReplace(TypedDict):
+    type: Literal["replace"]
+    host: Optional[str]
+    find: str
+    replace_with: str
+
+
+class LinkRewriteRegex(TypedDict):
+    type: Literal["regex"]
+    host: Optional[str]
+    pattern: str
+    replace_with: str
+
+
+LinkRewriteRule = Union[LinkRewriteReplace, LinkRewriteRegex]
+
+
 class RequestParamsDict(TypedDict, total=False):
     # The URL to be crawled.
     url: Optional[str]
@@ -281,6 +300,9 @@ class RequestParamsDict(TypedDict, total=False):
     # The user agent string to be used for the request.
     user_agent: Optional[str]
 
+    # URL rewrite rule applied to every discovered link before it's crawled.
+    link_rewrite: Optional[LinkRewriteRule]
+
     # The two letter country code for the request geo-location.
     country_code: Optional[str]
 

python/spider/spider_types.py

@@ -1,6 +1,6 @@
 [package]
 name = "spider-client"
-version = "0.1.80"
+version = "0.1.82"
 edition = "2021"
 authors = [ "j-mendez <jeff@spider.cloud>"]
 description = "Spider Cloud client"