B4J Question Http Sessions

[mbenam]

Hello all,

Is there an example on how we can use a single session to make multiple http request to a website? I understand that we need to send some of the cookies in subsequent requests. Looking for an example of such implementations.

Thanks.

 

[Erel]

Make sure to use (j)OkHttpUtils2. It will create a single session and manage the cookies for you.

 

[mbenam]

I tried the following. Supposed to return a json. Currently not getting anything.

'Non-UI application (console / server application)
#Region Project Attributes
    #CommandLineArgs:
    #MergeLibraries: True
#End Region

Sub Process_Globals
    
End Sub

Public Sub MidExtract (StringText As String, Start As String, EndAt As String) As String
    Dim i1 As Int = StringText.IndexOf(Start) + Start.Length
    Dim i2 As Int = StringText.IndexOf2(EndAt, i1)
    Return StringText.SubString2(i1, i2)
End Sub

Sub AppStart (Args() As String)
    getBarChart
    Log("Hello world!!!")
    StartMessageLoop
End Sub

Sub getBarChart()
    
    Dim xsrfToken As String
    Dim su As StringUtils
    Dim job As HttpJob
    job.Initialize("", Me)
    job.Download("https://www.barchart.com/futures/quotes/CLZ24/price-history/historical")
    job.GetRequest.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36")
    Wait For (job) JobDone(job As HttpJob)
    If job.Success Then

        Dim cookies As List
        cookies.Initialize
        cookies = job.Response.GetHeaders.Get("set-cookie")
        xsrfToken = cookies.Get(1).As(String)
        xsrfToken = su.DecodeUrl(MidExtract(xsrfToken, "XSRF-TOKEN=", ";"), "UTF8")
        Dim job2 As HttpJob
        job2.Initialize("", Me)
    
        job2.Download("https://www.barchart.com/proxies/core-api/v1/historical/get?symbol=CLZ24&fields=tradeTime.format(m/d/Y),openPrice,highPrice,lowPrice,lastPrice,priceChange,percentChange,volume,openInterest,symbolCode,symbolType&type=eod&orderBy=tradeTime&orderDir=desc&limit=65&meta=field.shortName,field.type,field.description&raw=1")
        job2.GetRequest.SetHeader("X-XSRF-TOKEN", xsrfToken)
        job2.GetRequest.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36")
        If job2.Success Then
            Log(job2.GetString)
        End If
        job2.Release
      
    End If
    job.Release
End Sub

 

[mbenam]

Here is a python equivalent that works.

import json
import requests
from urllib.parse import unquote


def getBarChart(apiUrl, getUrl):
    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:86.0) Gecko/20100101 Firefox/86.0"
    }

    with requests.Session() as s:
        # get all cookies
        s.get(
            getUrl,
            headers=headers,
        )
        # use one cookie as HTTP header
        headers["X-XSRF-TOKEN"] = unquote(s.cookies["XSRF-TOKEN"])
        return s.get(apiUrl, headers=headers).json()


apiUrl = "https://www.barchart.com/proxies/core-api/v1/historical/get?symbol=CLZ24&fields=tradeTime.format(m/d/Y),openPrice,highPrice,lowPrice,lastPrice,priceChange,percentChange,volume,openInterest,symbolCode,symbolType&type=eod&orderBy=tradeTime&orderDir=desc&limit=65&meta=field.shortName,field.type,field.description&raw=1"
getUrl= "https://www.barchart.com/futures/quotes/CLZ24/price-history/historical"


print(getBarChart(apiUrl, getUrl))

 

[Erel]

You are missing the second Wait For (job2) JobComplete(...)

 

[mbenam]

That was it. Stupid mistake on my part. Thanks for pointing out Erel. Here is the complete working code.

'Non-UI application (console / server application)
#Region Project Attributes
    #CommandLineArgs:
    #MergeLibraries: True
#End Region

Sub Process_Globals
    
End Sub

Public Sub MidExtract (StringText As String, Start As String, EndAt As String) As String
    Dim i1 As Int = StringText.IndexOf(Start) + Start.Length
    Dim i2 As Int = StringText.IndexOf2(EndAt, i1)
    Return StringText.SubString2(i1, i2)
End Sub

Sub AppStart (Args() As String)
    getBarChart
    Log("Hello world!!!")
    StartMessageLoop
End Sub

Sub getBarChart()
    
    Dim xsrfToken As String
    Dim su As StringUtils
    Dim job As HttpJob
    job.Initialize("", Me)
    job.Download("https://www.barchart.com/futures/quotes/CLZ24/price-history/historical")
    job.GetRequest.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36")
    Wait For (job) JobDone(job As HttpJob)
    If job.Success Then

        Dim cookies As List
        cookies.Initialize
        cookies = job.Response.GetHeaders.Get("set-cookie")
        xsrfToken = cookies.Get(1).As(String)
        xsrfToken = su.DecodeUrl(MidExtract(xsrfToken, "XSRF-TOKEN=", ";"), "UTF8")
        Dim job2 As HttpJob
        job2.Initialize("", Me)
    
        job2.Download("https://www.barchart.com/proxies/core-api/v1/historical/get?symbol=CLZ24&fields=tradeTime.format(m/d/Y),openPrice,highPrice,lowPrice,lastPrice,priceChange,percentChange,volume,openInterest,symbolCode,symbolType&type=eod&orderBy=tradeTime&orderDir=desc&limit=65&meta=field.shortName,field.type,field.description&raw=1")
        job2.GetRequest.SetHeader("X-XSRF-TOKEN", xsrfToken)
        job2.GetRequest.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36")
        Wait For (job2) JobDone(job2 As HttpJob)
        If job2.Success Then
            Log(job2.GetString)
        End If
        job2.Release
      
    End If
    job.Release
End Sub