How to retrieve all results from a paginated API

import aiohttp
import asyncio
from typing import Optional, List, Dict, Any


async def authenticate(url: str, headers: Dict[str, str]) -> Optional[str]:
    """Authenticate with the API and return the JWT token."""
    async with aiohttp.ClientSession() as session:
        async with session.post(url, headers=headers) as response:
            if response.status == 200:
                data = await response.json()
                api_jwt = data.get("access_token")
                print("Authentication successful.")
                return api_jwt
            else:
                print(f"Failed to authenticate: HTTP {response.status}")
                return None


async def fetch_all_pages(url: str, headers: Dict[str, str]) -> List[Any]:
    """Fetch all pages of data from the API, handling pagination and rate limiting."""
    results: List[Any] = []
    params = {"limit": 100}
    backoff_time = 1

    async with aiohttp.ClientSession() as session:
        while url:
            async with session.get(url, headers=headers, params=params) as response:
                if response.status == 429:
                    content = await response.text()
                    if "limit exceeded" in content.lower():
                        print("Usage quota exceeded. Aborting.")
                        break
                    elif "too many requests" in content.lower():
                        print(f"Rate limit exceeded. Retrying in {backoff_time} seconds.")
                        await asyncio.sleep(backoff_time)
                        backoff_time *= 2
                    else:
                        print(f"Unhandled 429 error: {content}")
                        break
                elif response.status != 200:
                    print(f"Failed to fetch data: HTTP {response.status}")
                    break
                else:
                    data = await response.json()
                    results.extend(data["results"])
                    print(f"Fetched {len(data['results'])} results from {url}")
                    url = data.get("next")
                    params = {}
                    backoff_time = 1

    return results


async def main():
    auth_url = "https://<API Server FQDN>/authenticate"
    auth_headers = {
        "Accept": "application/json",
        "User-Agent": "YourUserAgent/1.0",
        "Authorization": "Bearer <Your Public API Key>",
    }

    jwt = await authenticate(auth_url, auth_headers)
    if jwt:
        # Modify api_url to target the intended API
        api_url = "https://<API Server FQDN>/api/sales/v1/customer/"
        api_headers = {
            "Accept": "application/json",
            "X-Active-Store": "1",
            "User-Agent": "YourUserAgent/1.0",
            "Authorization": f"Bearer {jwt}",
        }

        all_results = await fetch_all_pages(api_url, api_headers)
        print(f"Total results fetched: {len(all_results)}")
    else:
        print("Failed to authenticate. No data fetched.")

# Run the main function
asyncio.run(main())

import axios, {AxiosError, AxiosResponse} from 'axios';

interface ApiResponse {
    results: any[];
    next?: string;
}
interface AuthHeaders {
  [key: string]: string; // This index signature allows any string key with a string value.
  Accept: string;
  'User-Agent': string;
  Authorization: string;
}

// Define an interface for the expected structure of the authentication response data
interface AuthResponseData {
  access_token: string; // Add other properties if the response contains more
}

// Extend the AxiosResponse interface with the specific data type for authentication
interface AuthResponse extends AxiosResponse {
  data: AuthResponseData;
}

// Define the function with the correct expected response type
async function authenticate(url: string, headers: AuthHeaders): Promise<string | null> {
  try {
    // Cast the response to AuthResponse for proper type checking
    const response = await axios.post<AuthResponseData>(url, {}, { headers }) as AuthResponse;
    if (response.status === 200) {
      console.log("Authentication successful.");
      return response.data.access_token; // TypeScript now knows that access_token exists on response.data
    } else {
      console.log(`Failed to authenticate: HTTP ${response.status}`, response.data);
      return null;
    }
  } catch (error) {
    handleError(error as AxiosError, "Authentication error");
    return null;
  }
}

async function fetchAllPages(url: string, headers: AuthHeaders): Promise<any[]> {
    let results: any[] = [];
    let params = { limit: 100 };
    let backoffTime = 1;

    while (url) {
        try {
            const response = await axios.get<ApiResponse>(url, { headers, params });
            results = [...results, ...response.data.results];
            console.log(`Fetched ${response.data.results.length} results from ${url}`);
            url = response.data.next || '';
            params = { limit: 100 }; // Reset to initial structure with 'limit'
            backoffTime = 1; // Reset backoff time after a successful request
        } catch (error: any) {
            // Correctly await the async function here
            const shouldContinue = await handleRateLimit(error, backoffTime);
            if (!shouldContinue) break;

            // If rate limit was hit, increment the backoff time
            if (axios.isAxiosError(error) && error.response?.status === 429) {
                backoffTime *= 2;
            }
        }
    }
    return results;
}


async function handleRateLimit(error: any, backoffTime: number): Promise<boolean> {
  if (axios.isAxiosError(error) && error.response?.status === 429) {
    // Check the message within the response data to determine the type of rate limit error
    const errorMessage: string = error.response.data.message;

    if (errorMessage.toLowerCase().includes("too many requests")) {
      console.log(`Rate limit exceeded: too many requests. Retrying in ${backoffTime} seconds.`);
      await new Promise(resolve => setTimeout(resolve, backoffTime * 1000));
      return true; // Return true to retry
    } else if (errorMessage.toLowerCase().includes("limit exceeded")) {
      console.log(`Quota exceeded: Daily limit reached. Aborting.`);
      return false; // Return false to abort
    }
  } else if (axios.isAxiosError(error)) {
    console.error(`Failed to fetch data: HTTP ${error.response?.status}`, error.response.data);
  } else {
    console.error(`Unexpected error: ${error}`);
  }
  return false; // Default return value for other errors or if no response status is 429
}


function handleError(error: AxiosError | Error, message: string) {
    if (axios.isAxiosError(error) && error.response) {
        console.error(`${message}: HTTP ${error.response.status}`, error.response.data);
    } else {
        console.error(`${message}: ${error.message}`);
    }
}

async function main() {
    const authUrl = 'https://<API Server FQDN>/authenticate';
    const authHeaders: AuthHeaders = {
        Accept: 'application/json',
        'User-Agent': 'YourUserAgent/1.0',
        Authorization: 'Bearer <Your Public API Key Goes Here>',
    };

    const jwt = await authenticate(authUrl, authHeaders);

    if (jwt) {
        const apiUrl = 'https://<API Server FQDN>/api/sales/v1/customer/';
        const apiHeaders =
            {...authHeaders, Authorization: `Bearer ${jwt}`, 'X-Active-Store': '1'};
        const allResults = await fetchAllPages(apiUrl, apiHeaders);
        console.log(`Total results fetched: ${allResults.length}`);
    } else {
        console.log('Failed to authenticate. No data fetched.');
    }
}

main().catch((error) => {
  console.error("An error occurred in the main function:", error);
});

using System.Text.Json;
using System.Net;

class Program
{
    static HttpClient client = new HttpClient();

    static async Task Main(string[] args)
    {

        var handler = new HttpClientHandler();
        if (handler.SupportsAutomaticDecompression)
        {
            handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        }
        client = new HttpClient(handler);

        string baseUrl = "https://api.toogoerp.net/authenticate";
        var authHeaders = new Dictionary<string, string>
        {
            ["Accept"] = "application/json",
            ["User-Agent"] = "YourUserAgent/1.0",
            ["Authorization"] = "Bearer <Your Public API Key Goes Here>"
        };

        var jwt = await Authenticate(baseUrl, authHeaders);
        if (jwt != null)
        {
            string apiUrl = "https://api.toogoerp.net/api/sales/v1/customer/";
            authHeaders["Authorization"] = $"Bearer {jwt}";
            var allResults = await FetchAllPages(apiUrl, authHeaders);
            Console.WriteLine($"Total results fetched: {allResults.Count}");
        }
        else
        {
            Console.WriteLine("Failed to authenticate. No data fetched.");
        }
    }

    static async Task<string> Authenticate(string url, Dictionary<string, string> headers)
    {
        client.DefaultRequestHeaders.Clear();
        foreach (var header in headers)
        {
            client.DefaultRequestHeaders.Add(header.Key, header.Value);
        }

        var response = await client.PostAsync(url, null);
        if (response.IsSuccessStatusCode)
        {
            var content = await response.Content.ReadAsStringAsync();
            using (var jsonDoc = JsonDocument.Parse(content))
            {
                var root = jsonDoc.RootElement;
                var accessToken = root.GetProperty("access_token").GetString();
                Console.WriteLine("Authentication successful.");
                return accessToken;
            }
        }
        else
        {
            Console.WriteLine($"Failed to authenticate: HTTP {response.StatusCode}");
            return null;
        }
    }

    static async Task<List<JsonElement>> FetchAllPages(string url, Dictionary<string, string> headers)
    {
        List<JsonElement> results = new List<JsonElement>();
        var backoffTime = 1;
        bool fetchNextPage = true;

        while (fetchNextPage)
        {
            client.DefaultRequestHeaders.Clear();
            foreach (var header in headers)
            {
                client.DefaultRequestHeaders.Add(header.Key, header.Value);
            }
            
            // Specify Active Store ID
            client.DefaultRequestHeaders.Add("X-Active-Store", "1");

            try
            {
                var response = await client.GetAsync(url);
                if (response.IsSuccessStatusCode)
                {
                    var content = await response.Content.ReadAsStringAsync();
                    using (var jsonDoc = JsonDocument.Parse(content))
                    {
                        var root = jsonDoc.RootElement;
                        var data = root.GetProperty("results").EnumerateArray();
                        while (data.MoveNext())
                        {
                            results.Add(data.Current);
                        }
                        Console.WriteLine($"Fetched {root.GetProperty("results").GetArrayLength()} results from {url}");
                        url = root.TryGetProperty("next", out var nextUrl) ? nextUrl.GetString() : string.Empty;
                        backoffTime = 1; // Reset backoff time after a successful request
                        fetchNextPage = !string.IsNullOrEmpty(url);
                    }
                }
                else if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
                {
                    Console.WriteLine($"Rate limit exceeded. Retrying in {backoffTime} seconds.");
                    await Task.Delay(backoffTime * 1000);
                    backoffTime *= 2;
                }
                else
                {
                    Console.WriteLine($"Failed to fetch data: HTTP {response.StatusCode}");
                    fetchNextPage = false;
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"An error occurred: {ex.Message}");
                fetchNextPage = false;
            }
        }

        return results;
    }
}