From e5c2a5b75afc66b2df266cd558e8115d569c72ae Mon Sep 17 00:00:00 2001 From: stensmo <141365386+stensmo@users.noreply.github.com> Date: Fri, 1 Dec 2023 16:07:56 +0100 Subject: [PATCH] Added sftpstat and general clean up --- Project.toml | 2 + README.md | 5 +- src/SFTPClient.jl | 2 +- src/SFTPImpl.jl | 214 ++++++++++++++++++++++++++++------------------ test/readme.txt | 11 +++ test/runtests.jl | 1 + test/setup.jl | 23 ++++- 7 files changed, 170 insertions(+), 88 deletions(-) create mode 100644 test/readme.txt diff --git a/Project.toml b/Project.toml index 76bda3a..aa719fc 100644 --- a/Project.toml +++ b/Project.toml @@ -11,6 +11,7 @@ NetworkOptions = "ca575930-c2e3-43a9-ace4-1e988b2c1908" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" [compat] ArgTools = "1.1" @@ -21,6 +22,7 @@ URIs = "1.5.0" julia = "1.9" CSV = "0.10" + [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" diff --git a/README.md b/README.md index f1a12f5..7c3f6c1 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Package for working with SFTP in Julia. Built on Downloads.jl, but in my opinion The SFTP client supports username/password as well as certificates for authentication. -The following methods are supported: readdir, download, upload, cd, rm, rmdir, mkdir, mv +The following methods are supported: readdir, download, upload, cd, rm, rmdir, mkdir, mv, sftpstat (like stat) Examples: @@ -16,6 +16,8 @@ Examples: downloadDir="/tmp/" SFTPClient.download.(sftp, files, downloadDir=downloadDir) + statStructs = stat(sftp) + ``` @@ -26,6 +28,7 @@ Examples: # For certificate authentication, you can do this (since 0.3.8) sftp = SFTP("sftp://mysitewhereIhaveACertificate.com", "myuser", "cert.pub", "cert.pem") + # The cert.pem is your certificate (private key), and the cert.pub can be obtained from the private key as following: ssh-keygen -y -f ./cert.pem. Save the output into "cert.pub". ``` diff --git a/src/SFTPClient.jl b/src/SFTPClient.jl index c61b8ac..fbe8383 100644 --- a/src/SFTPClient.jl +++ b/src/SFTPClient.jl @@ -1,6 +1,6 @@ module SFTPClient -export SFTP, readdir, download,upload, cd, rm, rmdir, mkdir, mv +export SFTP, readdir, download,upload, cd, rm, rmdir, mkdir, mv, sftpstat, SFTPStatStruct include("SFTPImpl.jl") diff --git a/src/SFTPImpl.jl b/src/SFTPImpl.jl index b3599a2..7575cb6 100644 --- a/src/SFTPImpl.jl +++ b/src/SFTPImpl.jl @@ -3,6 +3,7 @@ using Downloads using LibCURL using URIs using CSV +using Dates mutable struct SFTP downloader::Downloader @@ -20,9 +21,11 @@ end struct SFTPStatStruct desc::String mode :: UInt + nlink :: Int + uid :: String + gid :: String size :: Int64 mtime :: Float64 - end if Sys.iswindows() @@ -187,67 +190,7 @@ function SFTP(url::AbstractString, username::AbstractString, password::AbstractS return sftp end - -function reset_easy_hook(sftp::SFTP) - - downloader = sftp.downloader - - - downloader.easy_hook = (easy, info) -> begin - if sftp.username != nothing - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_USERNAME, sftp.username) - end - if sftp.password != nothing - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_PASSWORD, sftp.password) - end - - if sftp.disable_verify_host - - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_SSL_VERIFYHOST , 0) - end - - if sftp.disable_verify_peer - - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_SSL_VERIFYPEER , 1) - end - - - if sftp.public_key_file != nothing - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_SSH_PUBLIC_KEYFILE, sftp.public_key_file) - end - - if sftp.private_key_file != nothing - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_SSH_PRIVATE_KEYFILE, sftp.private_key_file) - end - - if sftp.verbose - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_VERBOSE, 1) - end - - Downloads.Curl.setopt(easy, CURLOPT_DIRLISTONLY, 1) - - - - - end -end - - -function handleRelativePath(fileName, sftp::SFTP) - baseUrl = sftp.uri - println("base url $baseUrl") - resolvedReference = resolvereference(baseUrl, escapeuri(fileName)) - fileName = resolvedReference.path - println(fileName) - return fileName -end - -function ftp_command(sftp::SFTP, command::String) - slist = Ptr{Cvoid}(0) - - slist = curl_slist_append(slist, command) - - sftp.downloader.easy_hook = (easy, info) -> begin +function setStandardOptions(sftp, easy, info) if sftp.username != nothing Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_USERNAME, sftp.username) end @@ -278,7 +221,40 @@ function ftp_command(sftp::SFTP, command::String) Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_VERBOSE, 1) end +end + +function reset_easy_hook(sftp::SFTP) + + downloader = sftp.downloader + + + downloader.easy_hook = (easy, info) -> begin + setStandardOptions(sftp, easy, info) + Downloads.Curl.setopt(easy, CURLOPT_DIRLISTONLY, 1) + + + + + end +end + +function handleRelativePath(fileName, sftp::SFTP) + baseUrl = sftp.uri + #println("base url $baseUrl") + resolvedReference = resolvereference(baseUrl, escapeuri(fileName)) + fileName = resolvedReference.path + #println(fileName) + return fileName +end + +function ftp_command(sftp::SFTP, command::String) + slist = Ptr{Cvoid}(0) + + slist = curl_slist_append(slist, command) + + sftp.downloader.easy_hook = (easy, info) -> begin + setStandardOptions(sftp, easy, info) Downloads.Curl.setopt(easy, CURLOPT_QUOTE, slist) end @@ -302,38 +278,100 @@ end Base.broadcastable(sftp::SFTP) = Ref(sftp) -function parseStat(stats::Vector{String}) - io = IOBuffer(); - +Base.isdir(st::SFTPStatStruct) = filemode(st) & 0xf000 == 0x4000 +isfile(st::SFTPStatStruct) = filemode(st) & 0xf000 == 0x8000 + +Base.filemode(st::SFTPStatStruct) = st.mode + +function parseDate(monthPart::String, dayPart::String, yearOrTimePart::String) + yearStr::String = occursin(":",yearOrTimePart) ? string(year(now())) : yearOrTimePart + timeStr::String = !occursin(":",yearOrTimePart) ? "00:00" : yearOrTimePart + + dateTime = DateTime("$monthPart $dayPart $yearStr $timeStr",dateformat"u d yyyy H:M ") + + return datetime2unix(dateTime) +end + +function parseMode(s::String)::UInt + length(s) != 10 && error("Not correct lenght") + + dirChar = s[1] + + dir = (dirChar == 'd') ? 0x4000 : 0x8000 + + owner = strToNumber(s[2:4]) + group = strToNumber(s[5:7]) + anyone = strToNumber(s[8:10]) + + return dir + owner * 8^2 + group * 8^1 + anyone * 8^0 + +end + +function strToNumber(s::String)::Int64 + b1 = (s[1] != '-') ? 4 : 0 + b2 = (s[2] != '-') ? 2 : 0 + b3 = (s[3] != '-') ? 1 : 0 + return b1+b2+b3 +end +function parseStat(s::String) - for i=1:length(s) + resultVec = Vector{String}(undef, 9) + + lastIndex = 1 + + parseCounter = 1 + + stringLength = length(s) + + i = 1 + + while (i < stringLength) c = s[i] if c == ' ' - if s[i-1] != ' ' - write(io, c) - end + resultVec[parseCounter] = s[lastIndex:i-1] + parseCounter += 1 + + while (i < stringLength && c == ' ') + i += 1 + c = s[i] + end + + lastIndex = i + + if parseCounter == 9 + resultVec[parseCounter] = s[lastIndex:end] + break + end - else - write(io, c) end + + i += 1 end + return resultVec - return io +end + + + +function makeStruct(stats::Vector{String})::SFTPStatStruct + SFTPStatStruct(stats[9], parseMode(stats[1]), parse(Int64, stats[2]), stats[3], stats[4], parse(Int64, stats[5]), parseDate(stats[6], stats[7], stats[8])) end -function Base.stat(sftp::SFTP, file::AbstractString) - #isdir(st::StatStruct) = filemode(st) & 0xf000 == 0x4000 +""" +sftpstat(sftp::SFTP) + +Like Julia stat, but returns a Vector of SFTPStatStructs. Note that you can only run this on directories. Can be used for checking if a file was modified, and much more. + +""" +function sftpstat(sftp::SFTP) + + sftp.downloader.easy_hook = (easy, info) -> begin - if sftp.username != nothing - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_USERNAME, sftp.username) - end - if sftp.password != nothing - Downloads.Curl.setopt(easy, Downloads.Curl.CURLOPT_PASSWORD, sftp.password) - end + setStandardOptions(sftp, easy, info) end @@ -346,20 +384,26 @@ function Base.stat(sftp::SFTP, file::AbstractString) sftp.uri = URI(uriString) end - dir = sftp.uri.path io = IOBuffer(); - output = Downloads.download(uriString, io; sftp.downloader) + try + output = Downloads.download(uriString, io; sftp.downloader) + + + finally + reset_easy_hook(sftp) + end + # Don't know why this is necessary res = String(take!(io)) io2 = IOBuffer(res) - files = readlines(io2;keep=false) + stats = readlines(io2;keep=false) - - return files + return makeStruct.(parseStat.(stats)) + #return files catch e rethrow() end diff --git a/test/readme.txt b/test/readme.txt new file mode 100644 index 0000000..b8a2f65 --- /dev/null +++ b/test/readme.txt @@ -0,0 +1,11 @@ +Welcome to test.rebex.net! + +You are connected to an FTP or SFTP server used for testing purposes +by Rebex FTP/SSL or Rebex SFTP sample code. Only read access is allowed. + +For information about Rebex FTP/SSL, Rebex SFTP and other Rebex libraries +for .NET, please visit our website at https://www.rebex.net/ + +For feedback and support, contact support@rebex.net + +Thanks! diff --git a/test/runtests.jl b/test/runtests.jl index b87745d..be89629 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,7 @@ include("setup.jl") @testset "Connect Test" begin @test files == actualFiles + @test stats == actualStructs @test isfile(tempDir * "KeyGenerator.png") @test dirs == ["example"] @test isfile("readme.txt") diff --git a/test/setup.jl b/test/setup.jl index 32f0ff9..60d6e93 100644 --- a/test/setup.jl +++ b/test/setup.jl @@ -5,6 +5,7 @@ using Test sftp = SFTP("sftp://test.rebex.net", "demo", "password") cd(sftp, "/pub/example") +stats = sftpstat(sftp) files = readdir(sftp) @@ -40,4 +41,24 @@ actualFiles = ["KeyGenerator.png", "pop3-console-client.png", "readme.txt", "winceclient.png", -"winceclientSmall.png"] \ No newline at end of file +"winceclientSmall.png"] +actualStructs = [ + + SFTPStatStruct(".", 0x00000000000041c0, 2, "demo", "users", 0, 1.6802208e9) + SFTPStatStruct("..", 0x00000000000041c0, 2, "demo", "users", 0, 1.6802208e9) + SFTPStatStruct("imap-console-client.png", 0x0000000000008100, 1, "demo", "users", 19156, 1.171584e9) + SFTPStatStruct("KeyGenerator.png", 0x0000000000008180, 1, "demo", "users", 36672, 1.1742624e9) + SFTPStatStruct("KeyGeneratorSmall.png", 0x0000000000008180, 1, "demo", "users", 24029, 1.1742624e9) + SFTPStatStruct("mail-editor.png", 0x0000000000008100, 1, "demo", "users", 16471, 1.171584e9) + SFTPStatStruct("mail-send-winforms.png", 0x0000000000008100, 1, "demo", "users", 35414, 1.171584e9) + SFTPStatStruct("mime-explorer.png", 0x0000000000008100, 1, "demo", "users", 49011, 1.171584e9) + SFTPStatStruct("pocketftp.png", 0x0000000000008180, 1, "demo", "users", 58024, 1.1742624e9) + SFTPStatStruct("pocketftpSmall.png", 0x0000000000008180, 1, "demo", "users", 20197, 1.1742624e9) + SFTPStatStruct("pop3-browser.png", 0x0000000000008100, 1, "demo", "users", 20472, 1.171584e9) + SFTPStatStruct("pop3-console-client.png", 0x0000000000008100, 1, "demo", "users", 11205, 1.171584e9) + SFTPStatStruct("readme.txt", 0x0000000000008180, 1, "demo", "users", 379, 1.69512912e9) + SFTPStatStruct("ResumableTransfer.png", 0x0000000000008180, 1, "demo", "users", 11546, 1.1742624e9) + SFTPStatStruct("winceclient.png", 0x0000000000008180, 1, "demo", "users", 2635, 1.1742624e9) + SFTPStatStruct("winceclientSmall.png", 0x0000000000008180, 1, "demo", "users", 6146, 1.1742624e9) + SFTPStatStruct("WinFormClient.png", 0x0000000000008180, 1, "demo", "users", 80000, 1.1742624e9) + SFTPStatStruct("WinFormClientSmall.png", 0x0000000000008180, 1, "demo", "users", 17911, 1.1742624e9)] \ No newline at end of file