+ }iC^RIHt^RIHtHt^RIHt^RIHt^RIH t H t H t H t H t ^RIHtHtHtHtHtHtHt^RIHt^RIHtHtHtHtHt^R IHt^R I H!t!^R I"H#t#^R I$H%t%] 'd ^R I&H't'H(t(H)t)!RR]4t*!RR]4t+!RR]]],]3,4t-]!]-4t.]!!R4!RR] 44t/]!!R4!RR]44t0!RR]04t1R#)) annotations)IteratorMutableMapping)datetime)IntEnum) TYPE_CHECKING AnnotatedAny TypedDictcast) BaseModelBeforeValidator ConfigDictFieldPlainSerializerPlainValidator TypeAdapter)URL)EnqueueStrategy HttpHeaders HttpMethod HttpPayloadJsonSerializable)crypto_random_object_id) docs_group)compute_unique_key)validate_http_url) NotRequiredRequiredSelfc:]tRt^tRt^t^t^t^t^t ^t ^t ^t Rt R#) RequestState(Crawlee-specific request handling state.N)__name__ __module__ __qualname____firstlineno____doc__ UNPROCESSED BEFORE_NAV AFTER_NAVREQUEST_HANDLERDONE ERROR_HANDLERERRORSKIPPED__static_attributes__r$b/Users/ahmad/.openclaw/workspace/my-crawler/.venv/lib/python3.14/site-packages/crawlee/_request.pyr"r"s-2KJIO DM EGr3r"c]tRt^"t$RtRtR]R&RtR]R&]Pt R]R&Rt R ]R &R t R ]R &Rt R]R&R tR]R&^tR]R&RtR]R&RtR#)CrawleeRequestData9Crawlee-specific configuration stored in the `user_data`.Nz=Annotated[int | None, Field(alias='maxRetries', frozen=True)] max_retrieszAAnnotated[EnqueueStrategy | None, Field(alias='enqueueStrategy')]enqueue_strategyr"statez:Annotated[int | None, Field(alias='sessionRotationCount')]session_rotation_countFz.Annotated[bool, Field(alias='skipNavigation')]skip_navigationz3Annotated[int | None, Field(alias='lastProxyTier')]last_proxy_tierzAnnotated[bool, Field()] forefrontz)Annotated[int, Field(alias='crawlDepth')] crawl_depthAnnotated[str | None, Field()] session_idr$)r%r&r'r(r)r8__annotations__r9r"r*r:r;r<r=r>r?rAr2r$r3r4r6r6"sCQUKNU[_W^?&22E<2:Y]V]DFKOCKKOOHO:*/I'/T=>K:>515J.58r3r6ca]tRt^At$Rt]!RR7t]!RR7tR] R&Rt R ] R &Rt R ] R &R Rlt RRlt RRltRRltRRltRV3RlltRRltRtV;t#)UserDatazRepresents the `user_data` part of a Request. Apart from the well-known attributes (`label` and `__crawlee`), it can also contain arbitrary JSON-compatible values. allow)extraF)initdict[str, JsonSerializable]__pydantic_extra__Nz>Annotated[CrawleeRequestData | None, Field(alias='__crawlee')] crawlee_datar@labelc V^8dQhRRRR/#)keystrreturnrr$)formats"r4 __annotate__UserData.__annotate__Qs,,s,'7,r3c (VPV,#NrIselfrNs&&r4 __getitem__UserData.__getitem__Qs&&s++r3c$V^8dQhRRRRRR/#)rMrNrOvaluerrPNoner$)rQs"r4rRrSTs"--s-+;--r3c VR8Xd,Ve"\V\4'g \R4hW nW PV&R#)rKNz`label` must be str or None) isinstancerO ValueErrorrKrI)rXrNr\s&&&r4 __setitem__UserData.__setitem__Ts: '> E3)?)? !>??J',$r3c V^8dQhRRRR/#)rMrNrOrPr]r$)rQs"r4rRrS]s))s)t)r3c VPVR#rUrVrWs&&r4 __delitem__UserData.__delitem__]s  # #C (r3cV^8dQhRR/#)rMrPz Iterator[str]r$)rQs"r4rRrS`s++-+r3c# :"VPRjxL R#L5irUrVrXs&r4__iter__UserData.__iter__`s****s cV^8dQhRR/#rMrPintr$)rQs"r4rRrScs,,,r3c ,\VP4#rU)lenrIris&r4__len__UserData.__len__cs4**++r3c V^8dQhRRRR/#)rMotherobjectrPboolr$)rQs"r4rRrSfsFtr3c <\V\4'd\SV` V4#\V\4'dVP 4V8H#\ #rU)r_r super__eq__dict model_dumpNotImplemented)rXrt __class__s&&r4ryUserData.__eq__fsD eY ' '7>%( ( eT " "??$- -r3cV^8dQhRR/#rmr$)rQs"r4rRrSos11#1r3c xVP4p\\\VP 4444#)z&Return hash based on the model fields.)r{hashtuplesorteditems)rXdatas& r4__hash__UserData.__hash__os) E&./00r3)rK)r%r&r'r(r)r model_configrrIrBrJrKrYrarerjrqryrr2 __classcell__)r}s@r4rDrDAsn G,L6;6G3GSWLPWC,0E )0),-)+,11r3rDOtherc]tRt^xt$RtR]R&R]R&R]R&R]R &R ]R &R ]R &R ]R &R ]R&R]R&R]R&R]R&R]R&R]R&R]R&R]R&RtR#)RequestOptionszOptions that can be used to customize request creation. This type exactly matches the parameters of `Request.from_url` method. z Required[str]urlzNotRequired[HttpMethod]methodz0NotRequired[HttpHeaders | dict[str, str] | None]headersz%NotRequired[HttpPayload | str | None]payloadzNotRequired[str | None]rKrA unique_keyidzNotRequired[bool]keep_url_fragmentuse_extended_unique_keyalways_enqueuez(NotRequired[dict[str, JsonSerializable]] user_datano_retryzNotRequired[EnqueueStrategy]r9zNotRequired[int | None]r8r$Nr%r&r'r(r)rBr2r$r3r4rrxsc  ## == 22 ""''''((..%%7722((r3rz Storage datacHa]tRt^t0o$Rt]!RRR7tR]R&R]R&RtR ]R &R t R ]R &] 'd] !4t R]R&/t R]R&M R]R&R]R&^tR]R&RtR]R&R tR]R&R tR]R&]R RRR R R RR RR RR RRR RR!RR"R R#R / R$R%ll4tR&R /R'R(llt]R)R*l4t]R+R,l4t]R-R.l4t]R/R0l4t]P4R1R2l4t]R3R4l4t]P4R5R6l4t]R7R8l4t]R9R:l4t]P4R;R<l4t]R=R>l4t]P4R?R@l4t]RARBl4t]P4RCRDl4t]RERFl4t ] P4RGRHl4t ]RIRJl4t!RKt"R #)LRequesta+Represents a request in the Crawlee framework, containing the necessary information for crawling operations. The `Request` class is one of the core components in Crawlee, utilized by various components such as request providers, HTTP clients, crawlers, and more. It encapsulates the essential data for executing web requests, including the URL, HTTP method, headers, payload, and user data. The user data allows custom information to be stored and persisted throughout the request lifecycle, including its retries. Key functionalities include managing the request's identifier (`id`), unique key (`unique_key`) that is used for request deduplication, controlling retries, handling state management, and enabling configuration for session rotation and proxy handling. The recommended way to create a new instance is by using the `Request.from_url` constructor, which automatically generates a unique key and identifier based on the URL and request parameters. ### Usage ```python from crawlee import Request request = Request.from_url('https://crawlee.dev') ``` T)validate_by_namevalidate_by_aliasz5Annotated[str, Field(alias='uniqueKey', frozen=True)]rzFAnnotated[str, BeforeValidator(validate_http_url), Field(frozen=True)]rGETz)Annotated[HttpMethod, Field(frozen=True)]rNzAnnotated[HttpPayload | None, BeforeValidator(lambda v: v.encode() if isinstance(v, str) else v), PlainSerializer(lambda v: v.decode() if isinstance(v, bytes) else v), Field(frozen=True)]rrrrHrz:Annotated[HttpHeaders, Field(default_factory=HttpHeaders)]a%Annotated[dict[str, JsonSerializable], Field(alias='userData', default_factory=UserData), PlainValidator(user_data_adapter.validate_python), PlainSerializer(lambda instance: user_data_adapter.dump_python(instance, by_alias=True, exclude_none=False, exclude_unset=True, exclude_defaults=True))]z)Annotated[int, Field(alias='retryCount')] retry_countFz'Annotated[bool, Field(alias='noRetry')]rzSAnnotated[str | None, BeforeValidator(validate_http_url), Field(alias='loadedUrl')] loaded_urlz4Annotated[datetime | None, Field(alias='handledAt')] handled_atrKrArrrr9r8cPV^8dQhRRRRRRRRR R R R R R R RRRRRRRRRRRRR/#)rMrrOrrrz#HttpHeaders | dict[str, str] | NonerzHttpPayload | str | NonerK str | NonerArrrvrrr9zEnqueueStrategy | Noner8 int | Nonekwargsr rPr r$)rQs"r4rRRequest.__annotate__scc c c 5 c * cccc c"&cc1c cc !cr3c  hVeV 'd \R4h\V\4'gVf\T;'g/4p\V\4'dVP 4pT;'g\ VVVVVVV R7pV 'd\4 RV 2pV PR/4;'g/pVPR/4pV eWR&V eWR&\R/VBpV'dVVR&V!RRVR VR VR VR VRV/V BpVeVVPR &VeVVPn V#)aCreate a new `Request` instance from a URL. This is recommended constructor for creating new `Request` instances. It generates a `Request` object from a given URL with additional options to customize HTTP method, payload, unique key, and other request properties. If no `unique_key` or `id` is provided, they are computed automatically based on the URL, method and payload. It depends on the `keep_url_fragment` and `use_extended_unique_key` flags. Args: url: The URL of the request. method: The HTTP method of the request. headers: The HTTP headers of the request. payload: The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests. label: A custom label to differentiate between request types. This is stored in `user_data`, and it is used for request routing (different requests go to different handlers). session_id: ID of a specific `Session` to which the request will be strictly bound. If the session becomes unavailable when the request is processed, a `RequestCollisionError` will be raised. unique_key: A unique key identifying the request. If not provided, it is automatically computed based on the URL and other parameters. Requests with the same `unique_key` are treated as identical. keep_url_fragment: Determines whether the URL fragment (e.g., `#section`) should be included in the `unique_key` computation. This is only relevant when `unique_key` is not provided. use_extended_unique_key: Determines whether to include the HTTP method, ID Session and payload in the `unique_key` computation. This is only relevant when `unique_key` is not provided. always_enqueue: If set to `True`, the request will be enqueued even if it is already present in the queue. Using this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`. enqueue_strategy: The strategy that will be used for enqueuing the request. max_retries: Maximum number of retries for this request. Allows to override the global `max_request_retries` option of `BasicCrawler`. **kwargs: Additional request properties. z:`always_enqueue` cannot be used with a custom `unique_key`)rrrrArr|r __crawlee maxRetriesenqueueStrategyrrrrrrKr$)r`r_rzrrOencoderrpopgetr6rrJrA)clsrrrrrKrArrrrr9r8ruser_data_dictcrawlee_data_dictrJrequests&&$$$$$$$$$$$, r4from_urlRequest.from_urlsx`  !nYZ Z gt $ $!'--R0G gs # #nn&G  #5 !/$;$  356a |DJK4::*..{B?  ".9l +  '3C/ 0)>,=> *6N; '  !      %     ).G  g &  !.8G +r3defaultc$V^8dQhRRRRRR/#)rMparamrOrrrPr$)rQs"r4rRrXs"00c0z0U_0r3c b\VP4PpVPW4#)z9Get the value of a specific query parameter from the URL.)rrqueryr)rXrr query_paramss&&$ r4get_query_param_from_url Request.get_query_param_from_urlXs&488}** //r3cV^8dQhRR/#rMrPrr$)rQs"r4rRr^s66z6r3c B\RVP4P#)z?A string used to differentiate between arbitrary request types.rD)r rrKris&r4rK Request.label]sJ/555r3cV^8dQhRR/#rr$)rQs"r4rRrcs,,J,r3c .VPP#)z-The ID of the bound session, if there is any.)rJrAris&r4rARequest.session_idbs  +++r3cV^8dQhRR/#)rMrPr6r$)rQs"r4rRrhs&&0&r3c \RVP4pVPf\4VnVP#)r7rD)r rrJr6)rXrs& r4rJRequest.crawlee_datags7T^^4  ! ! )%7%9I "%%%r3cV^8dQhRR/#rmr$)rQs"r4rRrqs--S-r3c .VPP#)z+The depth of the request in the crawl tree.rJr?ris&r4r?Request.crawl_depthp  ,,,r3c V^8dQhRRRR/#rM new_valuernrPr]r$)rQs"r4rRrvs22S2T2r3c &WPnR#rUrrXrs&&r4r?rus(1%r3cV^8dQhRR/#)rMrPr"r$)rQs"r4rRrzs''|'r3c .VPP#)r#rJr:ris&r4r: Request.stateys  &&&r3c V^8dQhRRRR/#)rM new_stater"rPr]r$)rQs"r4rRrs,,|,,r3c &WPnR#rUr)rXrs&&r4r:r~s"+r3cV^8dQhRR/#rMrPrr$)rQs"r4rRrs--Z-r3c .VPP#)z?Crawlee-specific limit on the number of retries of the request.)rJr8ris&r4r8Request.max_retriesrr3cV^8dQhRR/#rr$)rQs"r4rRrs88 8r3c .VPP#)zFCrawlee-specific number of finished session rotations for the request.rJr;ris&r4r;Request.session_rotation_counts  777r3c V^8dQhRRRR/#)rMnew_session_rotation_countrnrPr]r$)rQs"r4rRrsNNNNr3c &WPnR#rUr)rXrs&&r4r;rs3M0r3cV^8dQhRR/#)rMrPrr$)rQs"r4rRrs;;/;r3c BVPP;'gR#)z5The strategy that was used for enqueuing the request.allrJr9ris&r4r9Request.enqueue_strategys  11::U:r3c V^8dQhRRRR/#)rMnew_enqueue_strategyrrPr]r$)rQs"r4rRrsBB_BBr3c &WPnR#rUr)rXrs&&r4r9rs-A*r3cV^8dQhRR/#rr$)rQs"r4rRrs111r3c .VPP#)z0The last proxy tier used to process the request.rJr=ris&r4r=Request.last_proxy_tiers  000r3c V^8dQhRRRR/#rr$)rQs"r4rRrs6666r3c &WPnR#rUrrs&&r4r=rs,5)r3cV^8dQhRR/#rMrPrvr$)rQs"r4rRrs++4+r3c .VPP#)zJIndicate whether the request should be enqueued at the front of the queue.rJr>ris&r4r>Request.forefronts  ***r3c V^8dQhRRRR/#)rMrrvrPr]r$)rQs"r4rRrs0040D0r3c &WPnR#rUrrs&&r4r>rs&/#r3cV^8dQhRR/#rr$)rQs"r4rRrs++T+r3c VPRJ#)z*Indicates whether the request was handled.N)rris&r4was_already_handledRequest.was_already_handledsd**r3r$)#r%r&r'r(r)rrrBrrrrrrrrrr classmethodrrpropertyrKrArJr?setterr:r8r;r9r=r>rr2)__conditional_annotations__s@r4rrs.ttLLEE  PO9>F 5=    *},#13 .3 LK#   >?K:>78=H5=OfjJcjjGKJDK1c# c 8< c -1 c!c"&c"&c#(c).c %c48c#'ccJ0D0 66,,&&--22'' \\,,--88""N#N;;BB1166++00++r3rc&]tRtRt$RtR]R&RtR#)RequestWithLockiz0A crawling request with information about locks.z1Annotated[datetime, Field(alias='lockExpiresAt')]lock_expires_atr$Nrr$r3r4rrs:FF.r3rN)2 __future__rcollections.abcrrrenumrtypingrr r r r pydanticr rrrrrryarlrcrawlee._typesrrrrrcrawlee._utils.cryptorcrawlee._utils.docsrcrawlee._utils.requestsrcrawlee._utils.urlsrtyping_extensionsrrr r"r6rOrDuser_data_adapterrrrr$r3r4rs"4AApppbb9*61== 7 99>11y..>)>?11h ) G)Y)). N]+i]+]+@ /g/r3