13
13
14
14
import flask
15
15
import httpx
16
- from httpx import URL , HTTPStatusError , Limits , RequestError , Timeout
16
+ from httpx import URL , Limits , Timeout
17
17
from pydantic import ValidationError
18
18
from requests import Response
19
19
from requests .structures import CaseInsensitiveDict
118
118
from palace .manager .sqlalchemy .util import get_one
119
119
from palace .manager .util import base64
120
120
from palace .manager .util .datetime_helpers import utc_now
121
+ from palace .manager .util .http .async_http import AsyncClient
122
+ from palace .manager .util .http .base import ResponseCodesTypes
121
123
from palace .manager .util .http .exception import BadResponseException
122
124
from palace .manager .util .http .http import HTTP , RequestKwargs
123
125
@@ -704,32 +706,24 @@ async def fetch_book_info_list(
704
706
books ,
705
707
urls ,
706
708
)
707
- except (RequestError , HTTPStatusError ) as e :
708
- self .log .error (f"Request error: { e } " )
709
- self .log .error (f"URL: { e .request .url } " )
710
- request_url = str (e .request .url )
711
- retried_requests [request_url ] += 1
712
-
713
- if retried_requests [request_url ] > 3 :
714
- response .raise_for_status ()
709
+ except BadResponseException as e :
710
+ if e .response .status_code == 404 :
711
+ self .log .warning (
712
+ f"404 returned: { e .response .url } : ignoring..."
713
+ )
715
714
else :
716
- if "404 Not Found" in str (e ):
717
- self .log .warning (
718
- f'url "{ e .request .url } " NOT FOUND. Skipping...'
719
- )
720
- else :
721
- self .log .warning (
722
- f"Retrying request (attempt { retried_requests [request_url ]} /3)"
723
- )
724
- urls .appendleft (request_url )
715
+ self .log .error (f"Request error: { e } " )
716
+ e .response .raise_for_status ()
725
717
if urls :
726
718
self ._make_request (client , urls , pending_requests )
727
719
728
720
return list (books .values ()), next
729
721
730
- def create_async_client (self , connections : int = 5 ) -> httpx .AsyncClient :
731
- return httpx .AsyncClient (
722
+ def create_async_client (self , connections : int = 5 ) -> AsyncClient :
723
+ return AsyncClient .for_web (
724
+ max_retries = 3 ,
732
725
timeout = Timeout (20.0 , pool = None ),
726
+ allowed_response_codes = ResponseCodesTypes ,
733
727
limits = Limits (
734
728
max_connections = connections ,
735
729
max_keepalive_connections = connections ,
@@ -739,7 +733,7 @@ def create_async_client(self, connections: int = 5) -> httpx.AsyncClient:
739
733
740
734
def _make_request (
741
735
self ,
742
- client : httpx . AsyncClient ,
736
+ client : AsyncClient ,
743
737
urls : deque [str ],
744
738
pending_requests : list [asyncio .Task [httpx ._models .Response ]],
745
739
) -> None :
0 commit comments