Hi, All
I'm trying to develop HttpJob queue manager.
StackBufferUsage is started to rise when the connection is off\lost, and soon ESP32 MCU is panic rebooted.
The project is attached.
@Erel's HttpJob module is being updated:
The log output is interesting starting "success = 0", StackBufferUsage is starting to rise:
Team, help to develop the module - i'm sure it should be useful for any, as i think, if MCU Internet device is offline - no use to send any other HTTP_requests while the first one (and next) are not sent and received for sure from MCU device.
I'm trying to develop HttpJob queue manager.
StackBufferUsage is started to rise when the connection is off\lost, and soon ESP32 MCU is panic rebooted.
The project is attached.
@Erel's HttpJob module is being updated:
HttpJob module with synchronous queue:
'version 2.01 Erel's HttpJob module + Peacemaker's QueueManager based on Erel's GlobalStore module
Sub Process_Globals
Private requestCache(2500) As Byte
Private responseCache(14000) As Byte
Private responseIndex As Int
Private mJobname(32) As Byte
Private mVerb(8) As Byte
Private bc As ByteConverter
Private ssl As Boolean
Private port As Int
Private hostIndex, hostLen, pathIndex, pathLen, payloadIndex, payloadLen, headersIndex, headersLen As Int
Private astream As AsyncStreams
Private socket As WiFiSocket
Private sslsocket As WiFiSSLSocket
Public EOL() As Byte = Array As Byte(13, 10)
Type JobResult (JobName() As Byte, ErrorMessage() As Byte, Success As Boolean, Response() As Byte, Status As Int)
Private ResponseTimer As Timer
Public ResponseTimeout As UInt = 7000
Public WorkingFlag As Boolean
Public HttpErrorsCounter As ULong
'-------------------------------
'HTTP-requests queue storage based on GlobalStore module
'The data will be stored in this buffer.
'Change its size if you encounter out of bounds errors.
Private JobNamesBuffer(300) As Byte 'buffer for JabNames of the HTTP-requests
Private PayloadBuffer(10000) As Byte 'buffer for whole text of the HTTP-requests
Private mmSrcOffset, mmDestOffset, mmCount As UInt 'ignore
Private QueueSlotIndex As UInt = 0
Private QueueSize As Byte = 0
Private POSTurl() As Byte
'You can change the number of slots. You must update the next lines.
Private const QueueLimit As Byte = 8 'ignore hardcoded max qty in memory
Private JobNamesLengths(8) As Int
Private PayloadLengths(8) As Int
Private JobNames0(), JobNames1(), JobNames2(), JobNames3(), JobNames4(), JobNames5(), JobNames6(), JobNames7() As Byte
Private JobNameSlots() As Object = Array(JobNames0, JobNames1, JobNames2, JobNames3, JobNames4, JobNames5, JobNames6, JobNames7)
Private Payload0(), Payload1(), Payload2(), Payload3(), Payload4(), Payload5(), Payload6(), Payload7() As Byte
Private PayloadSlots() As Object = Array(Payload0, Payload1, Payload2, Payload3, Payload4, Payload5, Payload6, Payload7)
Private RequestTypes(8) As Byte '0/1 = GET/POST for each slot
End Sub
#Region ------Erel's HTTPJob module code--------------
Public Sub Initialize(JobName As String) As Boolean
bc.ArrayCopy(JobName, mJobname)
headersIndex = 0
headersLen = 0
ResponseTimer.Initialize("ResponseTimer_Tick", ResponseTimeout)
Return True
End Sub
Private Sub ResponseTimer_Tick
Log("Response timeout timer fired !")
ParseResult
End Sub
Public Sub AddHeader(Key() As Byte, Value() As Byte)
Dim b() As Byte = JoinBytes(Array(Key, ": ".GetBytes, Value, EOL))
bc.ArrayCopy2(b, 0, requestCache, headersIndex + headersLen, b.Length)
headersLen = headersLen + b.Length
End Sub
Public Sub Download (Link() As Byte)
ParseLink(Link, Null)
bc.ArrayCopy("GET", mVerb)
SendRequest(0)
End Sub
Public Sub Post (Link() As Byte, Payload() As Byte)
ParseLink(Link, Payload)
bc.ArrayCopy("POST", mVerb)
SendRequest(0)
End Sub
Public Sub Put (Link() As Byte, Payload() As Byte)
ParseLink(Link, Payload)
bc.ArrayCopy("PUT", mVerb)
SendRequest(0)
End Sub
Private Sub SendRequest (unused As Byte)
Dim host As String = bc.StringFromBytes(bc.SubString2(requestCache, hostIndex, hostIndex + hostLen))
Dim st As Stream = Null
'Log("trying to connect to: ", host, " port: ", port, " ssl: ", ssl)
If ssl Then
sslsocket.Close
If sslsocket.ConnectHost(host, port) Then
st = sslsocket.Stream
End If
Else
socket.Close
If socket.ConnectHost(host, port) Then
st = socket.Stream
End If
End If
If st = Null Then
SetError("Failed to connect")
Return
End If
Log("connected: ", host)
responseIndex = 0
astream.Initialize(st, "Astream_NewData", "Astream_Error")
astream.Write(mVerb).Write(" ").Write(bc.SubString2(requestCache, pathIndex, pathIndex + pathLen)).Write(" HTTP/1.0").Write(EOL)
'Log(mVerb," ", bc.SubString2(requestCache, pathIndex, pathIndex + pathLen)," HTTP/1.0")
astream.Write("Host: ").Write(host).Write(EOL)
astream.Write("Connection: close").Write(EOL)
Dim Payload() As Byte = bc.SubString2(requestCache, payloadIndex, payloadIndex + payloadLen)
If Payload.Length > 0 Then
astream.Write("Content-Length: ").Write(NumberFormat(Payload.Length, 0, 0)).Write(EOL)
End If
If headersLen > 0 Then
astream.Write(bc.SubString2(requestCache, headersIndex, headersIndex + headersLen))
End If
astream.Write(EOL)
If Payload.Length > 0 Then
astream.Write(Payload)
End If
End Sub
Private Sub AStream_NewData (Buffer() As Byte)
If ResponseTimer.Enabled = False Then ResponseTimer.Enabled = True
'Log("NewData: " , Buffer)
If responseIndex + Buffer.Length > responseCache.Length Then
Log("ResponseCache is full (", Buffer.Length, ")")
Return
End If
bc.ArrayCopy2(Buffer, 0, responseCache, responseIndex, Buffer.Length)
responseIndex = responseIndex + Buffer.Length
End Sub
Private Sub AStream_Error
ParseResult
End Sub
Private Sub ParseResult
ResponseTimer.Enabled = False
If responseIndex = 0 Then
SetError("Response not available.")
Return
End If
Dim response() As Byte = bc.SubString2(responseCache, 0, responseIndex)
Dim i As Int = bc.IndexOf(response, EOL)
Dim statusLine() As Byte = bc.SubString2(response, 0, i)
Dim i1 As Int = bc.IndexOf(statusLine, " ")
Dim i2 As Int = bc.IndexOf2(statusLine, " ", i1 + 1)
Dim status As Int = bc.StringFromBytes(bc.SubString2(statusLine, i1 + 1, i2))
If Floor(status / 100) = 3 Then 'handle redirections
i1 = bc.IndexOf(response, "Location:")
If i1 > -1 Then
i2 = bc.IndexOf2(response, EOL, i1 + 1)
Dim NewPath() As Byte = bc.Trim(bc.SubString2(response, i1 + 9, i2))
Log("Redirecting to: ", NewPath)
ParseLink(NewPath, bc.SubString2(requestCache, payloadIndex, payloadIndex + payloadLen))
CallSubPlus("SendRequest", 1, 0) 'to avoid stack overflows
Return
End If
End If
Dim jr As JobResult
jr.Success = Floor(status / 100) = 2
i = bc.IndexOf(response, Array As Byte(13, 10, 13, 10))
jr.Response = bc.SubString(response, i + 4)
jr.JobName = mJobname
jr.ErrorMessage = Array As Byte()
jr.Status = status
HttpErrorsCounter = 0 'reset
ProcessNextRequest(jr)
Main.JobDone(jr)
End Sub
Private Sub ParseLink (Link() As Byte, Payload() As Byte)
Dim hostStart As Int
If bc.StartsWith(Link, "https://") Then
ssl = True
hostStart = 8
Else if bc.StartsWith(Link, "http://") Then
ssl = False
hostStart = 7
Else
SetError("Invalid link")
Return
End If
Dim i As Int = bc.IndexOf2(Link, "/", hostStart)
Dim path() As Byte
If i = -1 Then
i = Link.Length
path = "/"
End If
Dim host() As Byte = bc.SubString2(Link, hostStart, i)
If i < Link.Length Then path = bc.SubString(Link, i)
Dim colonStart As Int = bc.IndexOf(host, ":")
If colonStart > -1 Then
port = bc.StringFromBytes(bc.SubString(host, colonStart + 1))
host = bc.SubString2(host, 0, colonStart)
Else
If ssl Then port = 443 Else port = 80
End If
SetRequestCache(host, path, Payload)
End Sub
Private Sub SetRequestCache(host() As Byte, path() As Byte, payload() As Byte)
If payload = Null Then payload = Array As Byte()
payloadIndex = headersIndex + headersLen
payloadLen = payload.Length
bc.ArrayCopy2(payload, 0, requestCache, payloadIndex, payloadLen)
hostIndex = payloadIndex + payloadLen
hostLen = host.Length
bc.ArrayCopy2(host, 0, requestCache, hostIndex, hostLen)
pathIndex = hostIndex + hostLen
pathLen = path.Length
bc.ArrayCopy2(path, 0, requestCache, pathIndex, pathLen)
End Sub
Private Sub SetError (msg() As Byte)
HttpErrorsCounter = HttpErrorsCounter + 1
Dim jr As JobResult
jr.JobName = mJobname
jr.ErrorMessage = msg
jr.Response = Array As Byte()
jr.Success = False
jr.Status = 0
ProcessNextRequest(jr)
Main.JobDone(jr)
End Sub
#End Region
#Region -----------QueueManager---------------------
'Add HTTP-request into the FIFO queue and start processing according to the queue
'RequestType = 0/1 = GET/POST for each slot
'URL is saved for GET request, used as the single API URL for all POST requests
'Payload: Null for GET request, payload text for POST request
Public Sub AddRequestToQueue(JobName As String, RequestType As Byte, URL() As Byte, Payload() As Byte) As Boolean
Log("QueueSize = ", QueueSize)
Dim isFullQueue As Boolean = QueueSize >= QueueLimit
If isFullQueue Then
Log("Queue is full, addition was ignored")
Else
QueueSize = QueueSize + 1
Log("Incremented QueueSize = ", QueueSize)
RequestTypes(QueueSize - 1) = RequestType
If RequestType = 0 Then 'GET
SaveJobNameSlot(QueueSize - 1, JobName)
SavePayloadSlot(QueueSize - 1, URL)
Else if RequestType = 1 Then 'POST
bc.ArrayCopy(URL, POSTurl) 'URL for all POST requests
SaveJobNameSlot(QueueSize - 1, JobName)
SavePayloadSlot(QueueSize - 1, Payload)
End If
End If
If WorkingFlag = False Then
ProcessNextRequest(Null)
Else
'Log("HTTPJob is already working, ignoring")
End If
Return Not(isFullQueue)
End Sub
Private Sub ProcessNextRequest (PreviosJob As JobResult)
Log("********ProcessNextRequest***********************")
WorkingFlag = False
'Log("WorkingFlag = ", WorkingFlag)
If PreviosJob <> Null Then 'previous request result
Log("----------------Prev JobName: ", PreviosJob.JobName, ", success = ", PreviosJob.Success)
If PreviosJob.Success = False Or PreviosJob.Status = 0 Then
Log("Prev Job ErrorMessage: ", PreviosJob.ErrorMessage)
're-try
Log("Re-trying HTTP-request: ", PreviosJob.JobName, "; QueueSlotIndex = ", QueueSlotIndex)
If HttpErrorsCounter > 5 Then
Log("HttpErrorsCounter = ", HttpErrorsCounter)
'CallSubPlus("restart", 3000, 0)
Main.AppStart
Return
End If
Else
'finished OK, go to next one
QueueSize = QueueSize - 1
Log("DEcremented QueueSize = ", QueueSize)
QueueSlotIndex = QueueSlotIndex + 1
If QueueSlotIndex > QueueLimit - 1 Then
QueueSlotIndex = 0
End If
Log("Incremented QueueSlotIndex = ", QueueSlotIndex)
End If
End If
Log("StackBufferUsage = ", StackBufferUsage)
If QueueSize = 0 Then 'queue is out, stop working
QueueSlotIndex = 0
Log("--------Queue is out, stop working--------")
Return
End If
WorkingFlag = True 'start working
Log("WorkingFlag = ", WorkingFlag)
Log("QueueSlotIndex = ", QueueSlotIndex)
'TODO: adding headers...
Select QueueSlotIndex
Case 0
Log("Payload0 = ", Payload0)
Initialize(bc.StringFromBytes(JobNames0))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload0)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload0)
End If
Case 1
Log("Payload1 = ", Payload1)
Initialize(bc.StringFromBytes(JobNames1))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload1)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload1)
End If
Case 2
Log("Payload2 = ", Payload2)
Initialize(bc.StringFromBytes(JobNames2))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload2)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload2)
End If
Case 3
Log("Payload3 = ", Payload3)
Initialize(bc.StringFromBytes(JobNames3))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload3)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload3)
End If
Case 4
Log("Payload4 = ", Payload4)
Initialize(bc.StringFromBytes(JobNames4))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload4)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload4)
End If
Case 5
Log("Payload5 = ", Payload5)
Initialize(bc.StringFromBytes(JobNames5))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload5)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload5)
End If
Case 6
Log("Payload6 = ", Payload6)
Initialize(bc.StringFromBytes(JobNames6))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload6)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload6)
End If
Case 7
Log("Payload7 = ", Payload7)
Initialize(bc.StringFromBytes(JobNames7))
If RequestTypes(QueueSlotIndex) = 0 Then 'GET
Download(Payload7)
Else if RequestTypes(QueueSlotIndex) = 1 Then 'POST
Put(POSTurl, Payload7)
End If
End Select
End Sub
Private Sub SavePayloadSlot(slot As Int, Value() As Byte)
Dim index As Int = 0
For i = 0 To slot - 1
index = index + PayloadLengths(i)
Next
Dim ToCopy As Int = 0
For i = slot + 1 To PayloadLengths.Length - 1
ToCopy = ToCopy + PayloadLengths(i)
Next
If PayloadLengths(slot) <> Value.Length Then
mmSrcOffset = index + PayloadLengths(slot)
mmDestOffset = index + Value.Length
mmCount = ToCopy
RunNative("PayloadMemMove", Null)
End If
Dim b As Byte = PayloadBuffer(index + Value.Length + ToCopy - 1) 'ignore (check for out of bounds)
bc.ArrayCopy2(Value, 0, PayloadBuffer, index, Value.Length)
PayloadLengths(slot) = Value.Length
mmSrcOffset = 0
For index = 0 To PayloadSlots.Length - 1
If index > 0 Then mmSrcOffset = mmSrcOffset + PayloadLengths(index - 1)
mmCount = PayloadLengths(index)
RunNative("PayloadSetSlot", PayloadSlots(index))
Next
End Sub
Private Sub SaveJobNameSlot(slot As Int, Value() As Byte)
Dim index As Int = 0
For i = 0 To slot - 1
index = index + JobNamesLengths(i)
Next
Dim ToCopy As Int = 0
For i = slot + 1 To JobNamesLengths.Length - 1
ToCopy = ToCopy + JobNamesLengths(i)
Next
If JobNamesLengths(slot) <> Value.Length Then
mmSrcOffset = index + JobNamesLengths(slot)
mmDestOffset = index + Value.Length
mmCount = ToCopy
RunNative("JobNameMemMove", Null)
End If
Dim b As Byte = JobNamesBuffer(index + Value.Length + ToCopy - 1) 'ignore (check for out of bounds)
bc.ArrayCopy2(Value, 0, JobNamesBuffer, index, Value.Length)
JobNamesLengths(slot) = Value.Length
mmSrcOffset = 0
For index = 0 To JobNameSlots.Length - 1
If index > 0 Then mmSrcOffset = mmSrcOffset + JobNamesLengths(index - 1)
mmCount = JobNamesLengths(index)
RunNative("JobNameSetSlot", JobNameSlots(index))
Next
End Sub
#if C
void PayloadSetSlot(B4R::Object* o) {
B4R::ArrayByte* ab = b4r_httpjob::_payloadbuffer;
B4R::ArrayByte* arr = (B4R::ArrayByte*)B4R::Object::toPointer(o);
arr->data = (Byte*)ab->data + b4r_httpjob::_mmsrcoffset;
arr->length = b4r_httpjob::_mmcount;
}
void PayloadMemMove(B4R::Object* o) {
B4R::ArrayByte* ab = b4r_httpjob::_payloadbuffer;
memmove((Byte*)ab->data + b4r_httpjob::_mmdestoffset,
(Byte*)ab->data + b4r_httpjob::_mmsrcoffset, b4r_httpjob::_mmcount);
}
void JobNameSetSlot(B4R::Object* o) {
B4R::ArrayByte* ab = b4r_httpjob::_jobnamesbuffer;
B4R::ArrayByte* arr = (B4R::ArrayByte*)B4R::Object::toPointer(o);
arr->data = (Byte*)ab->data + b4r_httpjob::_mmsrcoffset;
arr->length = b4r_httpjob::_mmcount;
}
void JobNameMemMove(B4R::Object* o) {
B4R::ArrayByte* ab = b4r_httpjob::_jobnamesbuffer;
memmove((Byte*)ab->data + b4r_httpjob::_mmdestoffset,
(Byte*)ab->data + b4r_httpjob::_mmsrcoffset, b4r_httpjob::_mmcount);
}
#End If
#End RegionThe log output is interesting starting "success = 0", StackBufferUsage is starting to rise:
What is difference when the Internet-connection is OK, non-stop downloading is here, and StackBufferUsage = 60 is fully OK ?
Team, help to develop the module - i'm sure it should be useful for any, as i think, if MCU Internet device is offline - no use to send any other HTTP_requests while the first one (and next) are not sent and received for sure from MCU device.
Last edited:
