GetURL source code
Posted: Tue Jun 01, 2004 7:16 pm
Code updated For 5.20+
This is the source code of the GETURL lib.
Anyway i've updated a lot, it supports cookies and file relocation, i've also added the geturlfilesize procedure...
This code can be improved, and if you do so, please post the update here!!!
This is the source code of the GETURL lib.
Anyway i've updated a lot, it supports cookies and file relocation, i've also added the geturlfilesize procedure...
This code can be improved, and if you do so, please post the update here!!!
Code: Select all
Structure filedata
url.s
savepath.s
buffer.l
proxy.s
puser.s
ppass.s
wwwuser.s
wwwpass.s
EndStructure
Global complete
ProcedureDLL.s GetUrlLastMod(*Struct.filedata)
url$=*Struct\url
buffer.l=*Struct\buffer
proxy$=*Struct\proxy
user$=*Struct\puser
pass$=*Struct\ppass
wuser$=*Struct\wwwuser
wpass$=*Struct\wwwpass
For a=1 To Len(url$)
s_start=FindString(url$,":",7)
s_end=FindString(url$,"/",s_start)
port$=Mid(url$,s_start+1,s_end-(s_start+1))
Next
For a=1 To Len(url$)
s_start=FindString(url$,"//",1)
s_end=FindString(url$,"/",s_start+2)
server$=Mid(url$,s_start+2,s_end-(s_start+2))
If port$<>""
ReplaceString(server$,":"+port$,"",1|2)
EndIf
Next
If port$<>""
port$="80"
EndIf
For a=Len(url$) To 1 Step -1
A$=Mid(url$,a,1)
If A$<>"/"
filename$=A$+filename$
Else
a=1
EndIf
Next
If proxy$<>""
For a=1 To Len(proxy$)
s_start=FindString(proxy$,":",7)
s_end=Len(proxy$)
proxyport$=Mid(proxy$,s_start+1,s_end)
Next
If proxyport$=""
proxyport$="3196"
EndIf
For a=1 To Len(proxy$)
A$=Mid(proxy$,a,1)
If A$<>":"
proxyserver$+A$
Else
a=Len(proxy$)
EndIf
Next
conc$=user$+":"+pass$
OutputBuffer = AllocateMemory(Len(conc$))
Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
penc$=PeekS(OutputBuffer)
EndIf
If wuser$<>""
conc$=wuser$+":"+wpass$
OutputBuffer = AllocateMemory(Len(conc$))
Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
wenc$=PeekS(OutputBuffer)
EndIf
header=#False
download=#False
file_size=0
If proxy$<>""
ConnectionID = OpenNetworkConnection(proxyserver$, Val(proxyport$))
Else
ConnectionID = OpenNetworkConnection(server$, Val(port$))
EndIf
If ConnectionID
;/// File Information
com$="HEAD "+url$+" HTTP/1.1"+Chr(13)+Chr(10)
com$+"Accept: */*"+Chr(13)+Chr(10)
com$+"Host: "+host$+Chr(13)+Chr(10)
com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
If proxy$<>""
com$+"Proxy-Authorization: Basic "+penc$+Chr(13)+Chr(10)
EndIf
If wuser$<>""
com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
EndIf
com$+Chr(13)+Chr(10)
res = SendNetworkData(ConnectionID,@com$,Len(com$))
Repeat
Delay(10)
Result = NetworkClientEvent(ConnectionID)
If Result=2 ;/// Raw data received
Content$ = Space(14500)
ReceiveNetworkData(ConnectionID,@Content$,14500)
Content$=Trim(Content$)
;/// File not found handle
If FindString(Content$,"404",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : 404 not found"
EndIf
;/// File moved
If FindString(Content$,"301",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : 301 moved"
EndIf
;/// www authorization required
If FindString(Content$,"401",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : www authorization required"
EndIf
;/// File found handle
If FindString(Content$,"200",1)
pos=FindString(LCase(Content$),"modified: " , 1)
pos1=FindString(Content$,Chr(13)+Chr(10),pos)
If pos
ProcedureReturn(Mid(Content$,pos+10,(pos1-pos-10)))
Else
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : no date"
EndIf
Else
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : 200 not found"
EndIf
header=#True
EndIf
Until header=#True
Else
ProcedureReturn "-1 : unable to connect"
EndIf
EndProcedure
ProcedureDLL.s GetUrlFileSize(*Struct.filedata)
url$=*Struct\url
buffer.l=*Struct\buffer
proxy$=*Struct\proxy
user$=*Struct\puser
pass$=*Struct\ppass
wuser$=*Struct\wwwuser
wpass$=*Struct\wwwpass
For a=1 To Len(url$)
s_start=FindString(url$,":",7)
s_end=FindString(url$,"/",s_start)
port$=Mid(url$,s_start+1,s_end-(s_start+1))
Next
For a=1 To Len(url$)
s_start=FindString(url$,"//",1)
s_end=FindString(url$,"/",s_start+2)
server$=Mid(url$,s_start+2,s_end-(s_start+2))
If port$<>""
ReplaceString(server$,":"+port$,"",1|2)
EndIf
Next
If port$<>""
port$="80"
EndIf
For a=Len(url$) To 1 Step -1
A$=Mid(url$,a,1)
If A$<>"/"
filename$=A$+filename$
Else
a=1
EndIf
Next
If proxy$<>""
For a=1 To Len(proxy$)
s_start=FindString(proxy$,":",7)
s_end=Len(proxy$)
proxyport$=Mid(proxy$,s_start+1,s_end)
Next
If proxyport$=""
proxyport$="3196"
EndIf
For a=1 To Len(proxy$)
A$=Mid(proxy$,a,1)
If A$<>":"
proxyserver$+A$
Else
a=Len(proxy$)
EndIf
Next
conc$=user$+":"+pass$
OutputBuffer = AllocateMemory(Len(conc$))
Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
penc$=PeekS(OutputBuffer)
EndIf
If wuser$<>""
conc$=wuser$+":"+wpass$
OutputBuffer = AllocateMemory(Len(conc$))
Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
wenc$=PeekS(OutputBuffer)
EndIf
header=#False
download=#False
file_size=0
If proxy$<>""
ConnectionID = OpenNetworkConnection(proxyserver$, Val(proxyport$))
Else
ConnectionID = OpenNetworkConnection(server$, Val(port$))
EndIf
If ConnectionID
;/// File Information
com$="HEAD "+url$+" HTTP/1.1"+Chr(13)+Chr(10)
com$+"Accept: */*"+Chr(13)+Chr(10)
com$+"Host: "+host$+Chr(13)+Chr(10)
com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
If proxy$<>""
com$+"Proxy-Authorization: Basic "+penc$+Chr(13)+Chr(10)
EndIf
If wuser$<>""
com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
EndIf
com$+Chr(13)+Chr(10)
res = SendNetworkData(ConnectionID,@com$,Len(com$))
Repeat
Delay(10)
Result = NetworkClientEvent(ConnectionID)
If Result=2 ;/// Raw data received
Content$ = Space(14500)
ReceiveNetworkData(ConnectionID,@Content$,14500)
Content$=Trim(Content$)
;/// File not found handle
If FindString(Content$,"404",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : 404 not found"
EndIf
;/// File moved
If FindString(Content$,"301",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : 301 moved"
EndIf
;/// www authorization required
If FindString(Content$,"401",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : www authorization required"
EndIf
;/// File found handle
If FindString(Content$,"200",1)
pos=FindString(Content$,"Content-Length:" , 1)
If pos
pos=FindString(Content$," " , pos+15)
file_size=Val(Mid(Content$,pos+1,Len(Content$)))
ProcedureReturn Str(file_size)
Else
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : no size"
EndIf
Else
CloseNetworkConnection(ConnectionID)
ProcedureReturn "-1 : 200 not found"
EndIf
header=#True
EndIf
Until header=#True
Else
ProcedureReturn "-1 : unable to connect"
EndIf
EndProcedure
ProcedureDLL GetUrlProgress()
ProcedureReturn complete
EndProcedure
Procedure GetUrlFile1(*Struct.filedata)
url$=*Struct\url
buffer.l=*Struct\buffer
If buffer<=0
buffer=10240
EndIf
proxy$=*Struct\proxy
user$=*Struct\puser
pass$=*Struct\ppass
wuser$=*Struct\wwwuser
wpass$=*Struct\wwwpass
For a=1 To Len(url$)
s_start=FindString(url$,":",7)
s_end=FindString(url$,"/",s_start)
port$=Mid(url$,s_start+1,s_end-(s_start+1))
Next
For a=1 To Len(url$)
s_start=FindString(url$,"//",1)
s_end=FindString(url$,"/",s_start+2)
server$=Mid(url$,s_start+2,s_end-(s_start+2))
If port$<>""
ReplaceString(server$,":"+port$,"",1|2)
EndIf
Next
If port$<>""
port$="80"
EndIf
For a=Len(url$) To 1 Step -1
A$=Mid(url$,a,1)
If A$<>"/"
filename$=A$+filename$
Else
a=1
EndIf
Next
If proxy$<>""
For a=1 To Len(proxy$)
s_start=FindString(proxy$,":",7)
s_end=Len(proxy$)
proxyport$=Mid(proxy$,s_start+1,s_end)
Next
If proxyport$=""
proxyport$="3196"
EndIf
For a=1 To Len(proxy$)
A$=Mid(proxy$,a,1)
If A$<>":"
proxyserver$+A$
Else
a=Len(proxy$)
EndIf
Next
conc$=user$+":"+pass$
OutputBuffer = AllocateMemory(Len(conc$))
Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
penc$=PeekS(OutputBuffer)
EndIf
If wuser$<>""
conc$=wuser$+":"+wpass$
OutputBuffer = AllocateMemory(Len(conc$))
Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
wenc$=PeekS(OutputBuffer)
EndIf
header=#False
download=#False
file_size=0
resend:
If proxy$<>""
ConnectionID = OpenNetworkConnection(proxyserver$, Val(proxyport$))
Else
ConnectionID = OpenNetworkConnection(server$, Val(port$))
EndIf
file$=ReplaceString(url$,"http://"+server$,"")
If ConnectionID
;/// File Information
com$="HEAD "+file$+" HTTP/1.1"+Chr(13)+Chr(10)
com$+"Accept: */*"+Chr(13)+Chr(10)
com$+"Host: "+server$+Chr(13)+Chr(10)
com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
If proxy$<>""
com$+"Proxy-Authorization: Basic "+enc$+Chr(13)+Chr(10)
EndIf
If wuser$<>""
com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
EndIf
If cookie$<>""
com$+"Cookie: "+cookie$+Chr(13)+Chr(10)
EndIf
If location$<>""
com$+"Location: "+location$+Chr(13)+Chr(10)
EndIf
com$+Chr(13)+Chr(10)
res = SendNetworkData(ConnectionID,@com$,Len(com$))
Repeat
Delay(10)
Result = NetworkClientEvent(ConnectionID)
If Result=2 ;/// Raw data received
Content$ = Space(14500)
ReceiveNetworkData(ConnectionID,@Content$,14500)
Content$=Trim(Content$)
;/// File not found handle
If FindString(Content$,"404",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn -1
EndIf
;/// File moved
If FindString(Content$,"301",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn -1
EndIf
;/// www authorization required
If FindString(Content$,"401",1)
CloseNetworkConnection(ConnectionID)
ProcedureReturn -1
EndIf
;/// File found handle but redirect
If FindString(Content$,"302",1)
location$=""
loc=FindString(Content$,"Location: ",1)
If loc>0
temploc.s=Mid(Content$,loc+10,Len(Content$))
For a=1 To Len(temploc)
tcok.s=Mid(temploc,a,1)
If tcok<>Chr(13)
location$+tcok
Else
Break 1
EndIf
Next
EndIf
EndIf
;/// Site sends cookie authentication
cok=FindString(Content$,"Set-Cookie: ",1)
cookie$=""
If cok>0
tempcok.s=Mid(Content$,cok+12,Len(Content$))
For a=1 To Len(tempcok)
tcok.s=Mid(tempcok,a,1)
If tcok<>";"
cookie$+tcok
Else
Break 1
EndIf
Next
Goto resend
EndIf
;/// File found handle
If FindString(Content$,"200",1)
pos=FindString(Content$,"Content-Length:" , 1)
If pos
pos=FindString(Content$," " , pos+15)
file_size=Val(Mid(Content$,pos+1,Len(Content$)))
If file_size<>0
filepos=Len(Content$)
per.f=100/file_size
download=#True
EndIf
Else
CloseNetworkConnection(ConnectionID)
ProcedureReturn -1
EndIf
header=#True
EndIf
EndIf
Until header=#True
If download=#True
;/// File Download
com$="GET "+file$+" HTTP/1.1"+Chr(13)+Chr(10)
com$+"Accept: */*"+Chr(13)+Chr(10)
com$+"Host: "+server$+Chr(13)+Chr(10)
com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
If proxy$<>""
com$+"Proxy-Authorization: Basic "+enc$+Chr(13)+Chr(10)
EndIf
If wuser$<>""
com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
EndIf
If cookie$<>""
com$+"Cookie: "+cookie$+Chr(13)+Chr(10)
EndIf
If location$<>""
com$+"Location: "+location$+Chr(13)+Chr(10)
EndIf
com$+Chr(13)+Chr(10)
res = SendNetworkData(ConnectionID,@com$,Len(com$))
If CreateFile(0,*Struct\savepath+filename$)
incoming_buffer=AllocateMemory(buffer)
Repeat
Delay(10)
Result = NetworkClientEvent(ConnectionID)
If Result=2 ;/// Raw data received
size=ReceiveNetworkData(ConnectionID,incoming_buffer,buffer)
received+size-filepos
If size>0
WriteData(0,incoming_buffer+filepos,size-filepos)
If filepos>0
filepos=0
EndIf
EndIf
complete=Int(received*per)
Delay(1)
If received=file_size
file=#True
EndIf
EndIf
Until file=#True
CloseFile(0)
CloseNetworkConnection(ConnectionID)
ProcedureReturn 0
EndIf
CloseNetworkConnection(ConnectionID)
EndIf
Else
ProcedureReturn -1
EndIf
EndProcedure
ProcedureDLL GetUrlFile(*Struct.filedata)
If CreateThread(@GetUrlFile1(),*Struct)
ProcedureReturn 1
Else
ProcedureReturn -1
EndIf
EndProcedure