Page 1 of 2

GetURL source code

Posted: Tue Jun 01, 2004 7:16 pm
by Num3
Code updated For 5.20+

This is the source code of the GETURL lib.

Anyway i've updated a lot, it supports cookies and file relocation, i've also added the geturlfilesize procedure...

This code can be improved, and if you do so, please post the update here!!!

Code: Select all

    Structure filedata
      url.s
      savepath.s
      buffer.l
      proxy.s
      puser.s
      ppass.s
      wwwuser.s
      wwwpass.s
    EndStructure

    Global complete


    ProcedureDLL.s GetUrlLastMod(*Struct.filedata)
     
      url$=*Struct\url
      buffer.l=*Struct\buffer
      proxy$=*Struct\proxy
      user$=*Struct\puser
      pass$=*Struct\ppass
      wuser$=*Struct\wwwuser
      wpass$=*Struct\wwwpass
     
      For a=1 To Len(url$)
        s_start=FindString(url$,":",7)
        s_end=FindString(url$,"/",s_start)
        port$=Mid(url$,s_start+1,s_end-(s_start+1))
      Next
     
      For a=1 To Len(url$)
        s_start=FindString(url$,"//",1)
        s_end=FindString(url$,"/",s_start+2)
        server$=Mid(url$,s_start+2,s_end-(s_start+2))
        If port$<>""
          ReplaceString(server$,":"+port$,"",1|2)
        EndIf
      Next
     
      If port$<>""
        port$="80"
      EndIf
     
      For a=Len(url$) To 1 Step -1
        A$=Mid(url$,a,1)
        If A$<>"/"
          filename$=A$+filename$
        Else
          a=1
        EndIf
      Next
     
      If proxy$<>""
       
        For a=1 To Len(proxy$)
          s_start=FindString(proxy$,":",7)
          s_end=Len(proxy$)
          proxyport$=Mid(proxy$,s_start+1,s_end)
        Next
       
        If proxyport$=""
          proxyport$="3196"
        EndIf
       
        For a=1 To Len(proxy$)
          A$=Mid(proxy$,a,1)
          If A$<>":"
            proxyserver$+A$
          Else
            a=Len(proxy$)
          EndIf
        Next
       
        conc$=user$+":"+pass$
        OutputBuffer = AllocateMemory(Len(conc$))
        Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
        penc$=PeekS(OutputBuffer)
      EndIf
     
      If wuser$<>""
        conc$=wuser$+":"+wpass$
        OutputBuffer = AllocateMemory(Len(conc$))
        Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
        wenc$=PeekS(OutputBuffer)
      EndIf
     
      header=#False
      download=#False
      file_size=0
     
      If proxy$<>""
        ConnectionID = OpenNetworkConnection(proxyserver$, Val(proxyport$))
      Else
        ConnectionID = OpenNetworkConnection(server$, Val(port$))
      EndIf
     
      If ConnectionID
       
        ;/// File Information
        com$="HEAD "+url$+" HTTP/1.1"+Chr(13)+Chr(10)
        com$+"Accept: */*"+Chr(13)+Chr(10)
        com$+"Host: "+host$+Chr(13)+Chr(10)
        com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
        If proxy$<>""
          com$+"Proxy-Authorization: Basic "+penc$+Chr(13)+Chr(10)
        EndIf
        If wuser$<>""
          com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
        EndIf
       
        com$+Chr(13)+Chr(10)
        res = SendNetworkData(ConnectionID,@com$,Len(com$))
       
        Repeat
          Delay(10)
          Result = NetworkClientEvent(ConnectionID)
         
          If Result=2 ;/// Raw data received
            Content$ = Space(14500)
            ReceiveNetworkData(ConnectionID,@Content$,14500)
           
            Content$=Trim(Content$)
           
            ;/// File not found handle
            If FindString(Content$,"404",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : 404 not found"
            EndIf
           
            ;/// File moved
            If FindString(Content$,"301",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : 301 moved"
            EndIf
           
            ;/// www authorization required
            If FindString(Content$,"401",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : www authorization required"
            EndIf
           
            ;/// File found handle
            If FindString(Content$,"200",1)
              pos=FindString(LCase(Content$),"modified: " , 1)
              pos1=FindString(Content$,Chr(13)+Chr(10),pos)
              If pos
                ProcedureReturn(Mid(Content$,pos+10,(pos1-pos-10)))
              Else
                CloseNetworkConnection(ConnectionID)
                ProcedureReturn "-1 : no date"
              EndIf
            Else
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : 200 not found"
            EndIf
            header=#True
          EndIf
         
        Until header=#True
       
      Else
        ProcedureReturn "-1 : unable to connect"
      EndIf
     
    EndProcedure

    ProcedureDLL.s GetUrlFileSize(*Struct.filedata)
     
      url$=*Struct\url
      buffer.l=*Struct\buffer
      proxy$=*Struct\proxy
      user$=*Struct\puser
      pass$=*Struct\ppass
      wuser$=*Struct\wwwuser
      wpass$=*Struct\wwwpass
     
      For a=1 To Len(url$)
        s_start=FindString(url$,":",7)
        s_end=FindString(url$,"/",s_start)
        port$=Mid(url$,s_start+1,s_end-(s_start+1))
      Next
     
      For a=1 To Len(url$)
        s_start=FindString(url$,"//",1)
        s_end=FindString(url$,"/",s_start+2)
        server$=Mid(url$,s_start+2,s_end-(s_start+2))
        If port$<>""
          ReplaceString(server$,":"+port$,"",1|2)
        EndIf
      Next
     
      If port$<>""
        port$="80"
      EndIf
     
      For a=Len(url$) To 1 Step -1
        A$=Mid(url$,a,1)
        If A$<>"/"
          filename$=A$+filename$
        Else
          a=1
        EndIf
      Next
     
      If proxy$<>""
       
        For a=1 To Len(proxy$)
          s_start=FindString(proxy$,":",7)
          s_end=Len(proxy$)
          proxyport$=Mid(proxy$,s_start+1,s_end)
        Next
       
        If proxyport$=""
          proxyport$="3196"
        EndIf
       
        For a=1 To Len(proxy$)
          A$=Mid(proxy$,a,1)
          If A$<>":"
            proxyserver$+A$
          Else
            a=Len(proxy$)
          EndIf
        Next
       
        conc$=user$+":"+pass$
        OutputBuffer = AllocateMemory(Len(conc$))
        Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
        penc$=PeekS(OutputBuffer)
      EndIf
     
      If wuser$<>""
        conc$=wuser$+":"+wpass$
        OutputBuffer = AllocateMemory(Len(conc$))
        Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
        wenc$=PeekS(OutputBuffer)
      EndIf
     
      header=#False
      download=#False
      file_size=0
     
      If proxy$<>""
        ConnectionID = OpenNetworkConnection(proxyserver$, Val(proxyport$))
      Else
        ConnectionID = OpenNetworkConnection(server$, Val(port$))
      EndIf
     
      If ConnectionID
       
        ;/// File Information
        com$="HEAD "+url$+" HTTP/1.1"+Chr(13)+Chr(10)
        com$+"Accept: */*"+Chr(13)+Chr(10)
        com$+"Host: "+host$+Chr(13)+Chr(10)
        com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
        If proxy$<>""
          com$+"Proxy-Authorization: Basic "+penc$+Chr(13)+Chr(10)
        EndIf
        If wuser$<>""
          com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
        EndIf
       
        com$+Chr(13)+Chr(10)
        res = SendNetworkData(ConnectionID,@com$,Len(com$))
       
        Repeat
          Delay(10)
          Result = NetworkClientEvent(ConnectionID)
         
          If Result=2 ;/// Raw data received
            Content$ = Space(14500)
            ReceiveNetworkData(ConnectionID,@Content$,14500)
           
            Content$=Trim(Content$)
           
            ;/// File not found handle
            If FindString(Content$,"404",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : 404 not found"
            EndIf
           
            ;/// File moved
            If FindString(Content$,"301",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : 301 moved"
            EndIf
           
            ;/// www authorization required
            If FindString(Content$,"401",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : www authorization required"
            EndIf
           
            ;/// File found handle
            If FindString(Content$,"200",1)
              pos=FindString(Content$,"Content-Length:" , 1)
              If pos
                pos=FindString(Content$," " , pos+15)
                file_size=Val(Mid(Content$,pos+1,Len(Content$)))
                ProcedureReturn Str(file_size)
              Else
                CloseNetworkConnection(ConnectionID)
                ProcedureReturn "-1 : no size"
              EndIf
            Else
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn "-1 : 200 not found"
            EndIf
            header=#True
          EndIf
         
        Until header=#True
       
      Else
        ProcedureReturn "-1 : unable to connect"
      EndIf
     
    EndProcedure
     
    ProcedureDLL GetUrlProgress()
      ProcedureReturn complete
    EndProcedure

    Procedure GetUrlFile1(*Struct.filedata)
      url$=*Struct\url
      buffer.l=*Struct\buffer
     
      If buffer<=0
        buffer=10240
      EndIf
     
      proxy$=*Struct\proxy
      user$=*Struct\puser
      pass$=*Struct\ppass
      wuser$=*Struct\wwwuser
      wpass$=*Struct\wwwpass
     
      For a=1 To Len(url$)
        s_start=FindString(url$,":",7)
        s_end=FindString(url$,"/",s_start)
        port$=Mid(url$,s_start+1,s_end-(s_start+1))
      Next
     
      For a=1 To Len(url$)
        s_start=FindString(url$,"//",1)
        s_end=FindString(url$,"/",s_start+2)
        server$=Mid(url$,s_start+2,s_end-(s_start+2))
        If port$<>""
          ReplaceString(server$,":"+port$,"",1|2)
        EndIf
      Next
     
      If port$<>""
        port$="80"
      EndIf
     
      For a=Len(url$) To 1 Step -1
        A$=Mid(url$,a,1)
        If A$<>"/"
          filename$=A$+filename$
        Else
          a=1
        EndIf
      Next
     
      If proxy$<>""
       
        For a=1 To Len(proxy$)
          s_start=FindString(proxy$,":",7)
          s_end=Len(proxy$)
          proxyport$=Mid(proxy$,s_start+1,s_end)
        Next
       
        If proxyport$=""
          proxyport$="3196"
        EndIf
       
        For a=1 To Len(proxy$)
          A$=Mid(proxy$,a,1)
          If A$<>":"
            proxyserver$+A$
          Else
            a=Len(proxy$)
          EndIf
        Next
       
        conc$=user$+":"+pass$
        OutputBuffer = AllocateMemory(Len(conc$))
        Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
        penc$=PeekS(OutputBuffer)
      EndIf
     
      If wuser$<>""
        conc$=wuser$+":"+wpass$
        OutputBuffer = AllocateMemory(Len(conc$))
        Base64Encoder(@conc$,Len(conc$),OutputBuffer,OutputBuffer*2)
        wenc$=PeekS(OutputBuffer)
      EndIf
     
      header=#False
      download=#False
      file_size=0
     
      resend:
     
      If proxy$<>""
        ConnectionID = OpenNetworkConnection(proxyserver$, Val(proxyport$))
      Else
        ConnectionID = OpenNetworkConnection(server$, Val(port$))
      EndIf
     
      file$=ReplaceString(url$,"http://"+server$,"")
     
      If ConnectionID
       
        ;/// File Information
        com$="HEAD "+file$+" HTTP/1.1"+Chr(13)+Chr(10)
        com$+"Accept: */*"+Chr(13)+Chr(10)
        com$+"Host: "+server$+Chr(13)+Chr(10)
        com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
        If proxy$<>""
          com$+"Proxy-Authorization: Basic "+enc$+Chr(13)+Chr(10)
        EndIf
        If wuser$<>""
          com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
        EndIf
        If cookie$<>""
          com$+"Cookie: "+cookie$+Chr(13)+Chr(10)
        EndIf
        If location$<>""
          com$+"Location: "+location$+Chr(13)+Chr(10)
        EndIf
       
        com$+Chr(13)+Chr(10)
        res = SendNetworkData(ConnectionID,@com$,Len(com$))
       
        Repeat
          Delay(10)
          Result = NetworkClientEvent(ConnectionID)
         
          If Result=2 ;/// Raw data received
            Content$ = Space(14500)
            ReceiveNetworkData(ConnectionID,@Content$,14500)
           
            Content$=Trim(Content$)
           
            ;/// File not found handle
            If FindString(Content$,"404",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn -1
            EndIf
           
            ;/// File moved
            If FindString(Content$,"301",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn -1
            EndIf
           
            ;/// www authorization required
            If FindString(Content$,"401",1)
              CloseNetworkConnection(ConnectionID)
              ProcedureReturn -1
            EndIf
           
            ;/// File found handle but redirect
            If FindString(Content$,"302",1)
              location$=""
              loc=FindString(Content$,"Location: ",1)
              If loc>0
                temploc.s=Mid(Content$,loc+10,Len(Content$))
                For a=1 To Len(temploc)
                  tcok.s=Mid(temploc,a,1)
                  If tcok<>Chr(13)
                    location$+tcok
                  Else
                    Break 1
                  EndIf
                Next
              EndIf
            EndIf
              ;/// Site sends cookie authentication
              cok=FindString(Content$,"Set-Cookie: ",1)
              cookie$=""
             
              If cok>0
                tempcok.s=Mid(Content$,cok+12,Len(Content$))
                For a=1 To Len(tempcok)
                  tcok.s=Mid(tempcok,a,1)
                  If tcok<>";"
                    cookie$+tcok
                  Else
                    Break 1
                  EndIf
                Next
                Goto resend
              EndIf

           
            ;/// File found handle
            If FindString(Content$,"200",1)
              pos=FindString(Content$,"Content-Length:" , 1)
              If pos
                pos=FindString(Content$," " , pos+15)
                file_size=Val(Mid(Content$,pos+1,Len(Content$)))
                If file_size<>0
                  filepos=Len(Content$)
                  per.f=100/file_size
                  download=#True
                EndIf
              Else
                CloseNetworkConnection(ConnectionID)
                ProcedureReturn -1
              EndIf
              header=#True
            EndIf
          EndIf
         
        Until header=#True
       
        If download=#True
          ;/// File Download
         
          com$="GET "+file$+" HTTP/1.1"+Chr(13)+Chr(10)
          com$+"Accept: */*"+Chr(13)+Chr(10)
          com$+"Host: "+server$+Chr(13)+Chr(10)
          com$+"User-Agent: PureDownload 1.0"+Chr(13)+Chr(10)
          If proxy$<>""
            com$+"Proxy-Authorization: Basic "+enc$+Chr(13)+Chr(10)
          EndIf
          If wuser$<>""
            com$+"Authorization: Basic "+wenc$+Chr(13)+Chr(10)
          EndIf
          If cookie$<>""
            com$+"Cookie: "+cookie$+Chr(13)+Chr(10)
          EndIf
          If location$<>""
            com$+"Location: "+location$+Chr(13)+Chr(10)
          EndIf
         
          com$+Chr(13)+Chr(10)
          res = SendNetworkData(ConnectionID,@com$,Len(com$))
         
          If CreateFile(0,*Struct\savepath+filename$)
            incoming_buffer=AllocateMemory(buffer)
           
            Repeat
              Delay(10)
             
              Result = NetworkClientEvent(ConnectionID)
             
              If Result=2 ;/// Raw data received
               
                size=ReceiveNetworkData(ConnectionID,incoming_buffer,buffer)
                received+size-filepos
                If size>0
                  WriteData(0,incoming_buffer+filepos,size-filepos)
                  If filepos>0
                    filepos=0
                  EndIf
                EndIf
               
                complete=Int(received*per)
               
                Delay(1)
               
                If received=file_size
                  file=#True
                EndIf
              EndIf
            Until file=#True
            CloseFile(0)
            CloseNetworkConnection(ConnectionID)
            ProcedureReturn 0
          EndIf
          CloseNetworkConnection(ConnectionID)
        EndIf
       
      Else
        ProcedureReturn -1
      EndIf
     
     
    EndProcedure
     
    ProcedureDLL GetUrlFile(*Struct.filedata)
      If CreateThread(@GetUrlFile1(),*Struct)
        ProcedureReturn 1
      Else
        ProcedureReturn -1
      EndIf
    EndProcedure

Posted: Tue Jun 01, 2004 8:27 pm
by RJP Computing
That looks great. Is it cross platform compatible? (Windows, Linux, ect.)

Posted: Tue Jun 01, 2004 8:46 pm
by Num3
Yes, it's just HTTP 1.1 specs and simple PB functions that work on win an linux and all future versions ;)

You just have to wait for version 3.90 for linux because of the memory commands !

Posted: Wed Jun 02, 2004 8:04 pm
by DEU.exe
Great work. :!:

Thanx.
:D

F.

Posted: Wed Jun 02, 2004 10:31 pm
by Dare2
Thanks Num3. :)

Posted: Thu Jun 03, 2004 12:35 am
by FloHimself
Hi,
getting an invalid page fault error when using functions with http proxy.
just added this to the example:

Code: Select all

urlfile\proxy="proxyip:proxyport"
urlfile\puser="foobar"
urlfile\ppass="123456789"
using winxp sp1, pb 3.90, squid/2.5.STABLE1

Posted: Thu Jun 03, 2004 8:19 am
by Num3
Are you using proxyip like this 123.456.789:6080 or myserverurl:6080 !?

Posted: Thu Jun 03, 2004 12:19 pm
by FloHimself
Using it like this:

Code: Select all

123.456.789.000:6080

Posted: Fri Aug 20, 2004 2:49 pm
by dell_jockey
Hi Num3!

great code, thank you soo much for publishing it! Did you ever try to have multiple threads process downloads through this lib? Is it thread-safe?

Posted: Sat Oct 30, 2004 11:04 pm
by johnnyutah
Can someone please explain how to use GetUrl?

I believe the right thing to do is to compile it to a DLL file but I'm not sure how to call the GetUrlLastMod procedure from my program - the *Struct.filedata part of the procedure is slightly confusing.

Sorry for all the questions recently - I'm still learning!

Posted: Sun Oct 31, 2004 7:33 am
by johnnyutah
Ok, here's the code I'm using - it's probably wrong as "Debug CallFunction(0, "GetUrlLastMod")" returns a value of 12386376!

Code: Select all

Structure filedata
url.s ; Url path 
savepath.s ; Local HD save path (no file name) 
buffer.l ; Internet incoming buffer 
proxy.s ; Proxy Address 
puser.s ; Proxy User name 
ppass.s ; Proxy Password 
wwwuser.s ; www user name
wwwpass.s ; www password 
EndStructure
NewList filesdata.filedata()

AddElement(filesdata())
  filesdata()\url = "http://www.purebasic.com/download/PureBasic_Demo.exe"
  filesdata()\savepath = "f:\downloads"
  filesdata()\buffer = 0
  filesdata()\proxy = ""
  filesdata()\puser = ""
  filesdata()\ppass = ""
  filesdata()\wwwuser = ""
  filesdata()\wwwpass = ""
ResetList(filesdata())
NextElement(filesdata())
If OpenLibrary(0, "GetUrl.dll")
 Debug CallFunction(0, "GetUrlLastMod")
 CloseLibrary(0)
EndIf
[/code]

..

Posted: Sun Oct 31, 2004 1:32 pm
by NoahPhense
That value is a pointer to a string. Use this:

Debug PeekS(CallFunction(0, "GetUrlLastMod"))



But I don't use a DLL, I use a library. Here it is if you want it:
GetUrl Lib

Just put it in your:
C:\Program Files\PureBasic\PureLibraries\UserLibraries\

And restart PureBasic.. here is a working example using the lib
(not the dll)..

Code: Select all


;/// This structure is required to send the data to the GetUrl Command Set
Structure filedata
  url.s ; Url path
  savepath.s ; Local HD save path (no file name)
  buffer.l ; Internet incoming buffer
  proxy.s ; Proxy Address
  puser.s ; Proxy User name
  ppass.s ; Proxy Password
  wwwuser.s ; www user name
  wwwpass.s ; www password
EndStructure

urlfile.filedata
urlfile\url="http://www.sedtech.com/isedquickpdf/downloads/4.35/iSQP0435DLL.zip"
urlfile\savepath="" ; blank equals same path as exe
urlfile\buffer=10240

If InitNetwork()
  
  OpenConsole()
  PrintN("***Getting file date***")
  PrintN("Last Modified Date: "+GetUrlLastMod(@urlfile))
  PrintN("<any key to continue>")
  PrintN(" ")
  Input()
  
  
  PrintN("***Getting file***")
  If GetUrlFile(@urlfile)
    ConsoleLocate(1,5)
    Print("Getting : "+urlfile\url)
    
    Repeat
      progress.l=GetUrlProgress() ;/// Get current file progress
      ConsoleLocate(1,6)
      Print("Complete: "+Str(progress)+"%")
      Delay(10)
    Until progress=100
    
    PrintN(" ")
    PrintN(" ")
    PrintN("Done! <any key to end>")
    
  EndIf
  
  Input()
  CloseConsole()
EndIf

End
Hope this helps.

- np

Posted: Sun Oct 31, 2004 5:35 pm
by johnnyutah
Thank you so much NoahPhense!

One of the things that's great about PureBasic is the support that users can get on these forums. I know it can be annoying if people continually post and in effect don't write their own code but when you're learning getting help from people like yourself is invaluable. Your help is very much appreciated.

..

Posted: Sun Oct 31, 2004 5:37 pm
by NoahPhense
Many people helped me get started. And to this day, these same people
continue to help and assist.

I'm only giving back what I've been given.

- np

Posted: Tue Dec 07, 2004 8:27 pm
by GPI
Found some bugs (it was possible, that in the content-string a Tag is set and in this tag include "401" -> error without error...) User-authorization work, but i can't try the proxi-code:

Code: Select all

outdated code...