From d08636bdad4f36431bfdf07e8400d7188a2df781 Mon Sep 17 00:00:00 2001 From: Tomas Bzatek Date: Fri, 25 Oct 2024 11:32:12 +0200 Subject: Rework file copy/open-read-write-close data handling Stick to the POSIX open(2), read(2), write(2), close(2) return values semantics, apply on the VFS interface. Also handle short reads and writes that are common for some gvfs backends. This makes cross-VFS copy work. --- UConfig.pas | 4 +- UCoreWorkers.pas | 71 ++++++++++++++----------- UEngines.pas | 140 +++++++++++++++++++++++++++++-------------------- ULibc.pas | 31 ++++++++--- vfs/uVFSprototypes.pas | 19 ++++--- 5 files changed, 159 insertions(+), 106 deletions(-) diff --git a/UConfig.pas b/UConfig.pas index 7f1cabb..d01ae5a 100644 --- a/UConfig.pas +++ b/UConfig.pas @@ -25,8 +25,8 @@ uses Classes, ULocale; resourcestring ConstAppTitle = 'Tux Commander'; - ConstAboutVersion = '0.6.82-dev'; - ConstAboutBuildDate = '2024-10-23'; + ConstAboutVersion = '0.6.83-dev'; + ConstAboutBuildDate = '2024-10-25'; {$IFDEF FPC} {$INCLUDE fpcver.inc} diff --git a/UCoreWorkers.pas b/UCoreWorkers.pas index 507e2b7..75a9453 100644 --- a/UCoreWorkers.pas +++ b/UCoreWorkers.pas @@ -1019,11 +1019,12 @@ var DefResponse: integer; // Global variables for this function var fsrc, fdst: TEngineFileDes; BSize: integer; Buffer: Pointer; - BytesDone, BytesRead, BytesWritten: Int64; + BytesDone, BytesRead, BytesWritten, BytesRemaining: Int64; LocalError: PGError; begin Result := False; LocalError := nil; + BytesDone := 0; DebugMsg(['ManualCopyFile: ', SourceFile, ' ---> ', DestFile]); fsrc := SrcEngine.OpenFile(SourceFile, omRead, @LocalError); if fsrc = nil then begin @@ -1042,7 +1043,6 @@ var DefResponse: integer; // Global variables for this function Exit; end; - BytesDone := 0; BSize := DestEngine.GetBlockSize; Buffer := malloc(BSize); if Buffer = nil then begin @@ -1050,60 +1050,66 @@ var DefResponse: integer; // Global variables for this function CopyFilesWorker_ProgressFunc(Self, 0, Error^); // Memory allocation failed Exit; end; - memset(Buffer, 0, BSize); - BytesWritten := 0; repeat + // Read block BytesRead := SrcEngine.ReadFile(fsrc, Buffer, BSize, @LocalError); - if (BytesRead = 0) and (Error^ <> nil) then begin + if BytesRead < 0 then begin g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_READ), LocalError^.message); g_error_free(LocalError); LocalError := nil; - Result := CopyFilesWorker_ProgressFunc(Self, BytesWritten, Error^); // Cannot read from source file + Result := CopyFilesWorker_ProgressFunc(Self, BytesDone, Error^); // Cannot read from source file if Result then begin g_error_free(Error^); Error^ := nil; Continue; end else Break; end; + + // Write block if BytesRead > 0 then begin - BytesWritten := DestEngine.WriteFile(fdst, Buffer, BytesRead, @LocalError); - if (BytesWritten < BytesRead) then begin + BytesRemaining := BytesRead; + repeat + BytesWritten := DestEngine.WriteFile(fdst, Buffer + (BytesRead - BytesRemaining), BytesRemaining, @LocalError); + if BytesWritten > 0 then + BytesRemaining := BytesRemaining - BytesWritten; + until (BytesRemaining = 0) or (BytesWritten <= 0); + if BytesWritten < 0 then begin g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_WRITE), LocalError^.message); g_error_free(LocalError); LocalError := nil; - Result := CopyFilesWorker_ProgressFunc(Self, BytesWritten, Error^); // Cannot write to target file - if Result then begin - g_error_free(Error^); - Error^ := nil; - Continue; - end else Break; + Result := False; + CopyFilesWorker_ProgressFunc(Self, BytesDone, Error^); // Cannot write to target file + Break; end; end; + // BytesRead == 0 means EOF BytesDone := BytesDone + BytesRead; Result := CopyFilesWorker_ProgressFunc(Self, BytesDone, nil); if not Result then Break; - until (BytesRead = 0) or (BytesWritten < BytesRead); + until BytesRead <= 0; libc_free(Buffer); - if not DestEngine.CloseFile(fdst, @LocalError) then begin - g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_CLOSE), LocalError^.message); - g_error_free(LocalError); - Result := False; - SrcEngine.CloseFile(fsrc, nil); - CopyFilesWorker_ProgressFunc(Self, BytesDone, Error^); // Cannot close target file - Exit; - end; - if not SrcEngine.CloseFile(fsrc, @LocalError) then begin - g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_CLOSE), LocalError^.message); - g_error_free(LocalError); - Result := CopyFilesWorker_ProgressFunc(Self, BytesDone, Error^); // Cannot close source file + if not DestEngine.CloseFile(fdst, @LocalError) then if Result then begin - // user has chosen to ignore the error - g_error_free(Error^); - Error^ := nil; - end else Exit; - end; + g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_CLOSE), LocalError^.message); + g_error_free(LocalError); + Result := False; + SrcEngine.CloseFile(fsrc, nil); + CopyFilesWorker_ProgressFunc(Self, BytesDone, Error^); // Cannot close target file + Exit; + end; + if not SrcEngine.CloseFile(fsrc, @LocalError) then + if Result then begin + g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_CLOSE), LocalError^.message); + g_error_free(LocalError); + Result := CopyFilesWorker_ProgressFunc(Self, BytesDone, Error^); // Cannot close source file + if Result then begin + // user has chosen to ignore the error + g_error_free(Error^); + Error^ := nil; + end else Exit; + end; end; // Returns True if the file was successfully copied and will be deleted on move @@ -1120,6 +1126,7 @@ var DefResponse: integer; // Global variables for this function then Result := DestEngine.CopyFileIn(SourceFile, DestFile, Append, @CopyFilesWorker_ProgressFunc, Self, Error) else + // TODO: check for reported Append capability and fall back to manual copy if needed // from local engine to VFS engine if (SrcEngine is TLocalTreeEngine) and (DestEngine is TVFSEngine) then begin diff --git a/UEngines.pas b/UEngines.pas index 8a0e9e6..8649a05 100644 --- a/UEngines.pas +++ b/UEngines.pas @@ -30,6 +30,11 @@ const omRead = 0; ConfDefaultDirCreationMask = 755; + ConfDefaultOpenFlagsRead = O_RDONLY or O_NOFOLLOW or O_CLOEXEC; + ConfDefaultOpenFlagsWrite = O_WRONLY or O_NOFOLLOW or O_CLOEXEC or O_EXCL or O_CREAT or O_TRUNC; + ConfDefaultOpenFlagsAppend = O_WRONLY or O_NOFOLLOW or O_CLOEXEC or O_EXCL or O_APPEND; + + ConfDefaultFileCreationMask = S_IRUSR or S_IWUSR or S_IRGRP or S_IROTH; type PDataItem = ^TDataItem; @@ -549,27 +554,27 @@ begin end; function TLocalTreeEngine.CopyFileOut(const SourceFile, DestFile: string; Append: boolean; ProgressFunc: TEngineProgressFunc; Sender: Pointer; Error: PPGError): boolean; -var fsrc, fdest: PFILE; +var fsrc, fdest: Longint; Buffer: Pointer; - BytesDone, BytesRead, BytesWritten: Int64; + BytesDone, BytesRead, BytesWritten, BytesRemaining: ssize_t; begin Result := False; BytesDone := 0; try // Open source file for reading - fsrc := fopen64(PChar(SourceFile), 'r'); - if fsrc = nil then begin + fsrc := open64(PChar(SourceFile), ConfDefaultOpenFlagsRead, 0); + if fsrc < 0 then begin g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_OPEN), '%m'); if @ProgressFunc <> nil then ProgressFunc(Sender, 0, Error^); Exit; end; // Open target file for writing/appending - if Append then fdest := fopen64(PChar(DestFile), 'a') - else fdest := fopen64(PChar(DestFile), 'w'); - if fdest = nil then begin + if Append then fdest := open64(PChar(DestFile), ConfDefaultOpenFlagsAppend, 0) + else fdest := open64(PChar(DestFile), ConfDefaultOpenFlagsWrite, ConfDefaultFileCreationMask); + if fdest < 0 then begin g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_OPEN), '%m'); - fclose(fsrc); + libc_close(fsrc); if @ProgressFunc <> nil then ProgressFunc(Sender, 0, Error^); Exit; @@ -578,17 +583,18 @@ begin Buffer := malloc(FBlockSize); if Buffer = nil then begin g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_ALLOC_FAILED), '%m'); - fclose(fdest); - fclose(fsrc); + libc_close(fdest); + libc_close(fsrc); if @ProgressFunc <> nil then ProgressFunc(Sender, 0, Error^); Exit; end; - while feof(fsrc) = 0 do begin + Result := True; + repeat // Read block - BytesRead := fread(Buffer, 1, FBlockSize, fsrc); - if (BytesRead < FBlockSize) and (feof(fsrc) = 0) then begin + BytesRead := libc_read(fsrc, Buffer, FBlockSize); + if BytesRead < 0 then begin g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_READ), '%m'); Result := False; if @ProgressFunc <> nil then @@ -602,41 +608,51 @@ begin end; // Write block - BytesWritten := fwrite(Buffer, 1, BytesRead, fdest); - if BytesWritten < BytesRead then begin - g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_WRITE), '%m'); - Result := False; - if @ProgressFunc <> nil then - ProgressFunc(Sender, BytesDone + BytesRead, Error^); - Break; // We cannot ignore write errors + if BytesRead > 0 then begin + BytesRemaining := BytesRead; + repeat + BytesWritten := libc_write(fdest, Buffer + (BytesRead - BytesRemaining), BytesRemaining); + if BytesWritten > 0 then + BytesRemaining := BytesRemaining - BytesWritten; + until (BytesRemaining = 0) or (BytesWritten <= 0); + if BytesWritten < 0 then begin + g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_WRITE), '%m'); + Result := False; + if @ProgressFunc <> nil then + ProgressFunc(Sender, BytesDone + BytesRead, Error^); + Break; // We cannot ignore write errors + end; end; - + // BytesRead == 0 means EOF BytesDone := BytesDone + BytesRead; - if (@ProgressFunc <> nil) and (not ProgressFunc(Sender, BytesDone, nil)) then + if (@ProgressFunc <> nil) and (not ProgressFunc(Sender, BytesDone, nil)) then begin + Result := False; Break; - end; - Result := (feof(fsrc) <> 0) and (Error^ = nil); + end; + until BytesRead <= 0; libc_free(Buffer); - if fclose(fdest) <> 0 then begin - g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_CLOSE), '%m'); - Result := False; - if @ProgressFunc <> nil then - ProgressFunc(Sender, BytesDone, Error^); - fclose(fsrc); - Exit; - end; - if fclose(fsrc) <> 0 then begin - g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_CLOSE), '%m'); - Result := False; - if @ProgressFunc <> nil then - Result := ProgressFunc(Sender, BytesDone, Error^); + if libc_close(fdest) <> 0 then if Result then begin - // user has chosen to ignore the error - g_error_free(Error^); - Error^ := nil; - end else Exit; - end; + g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_TARGET_CLOSE), '%m'); + Result := False; + if @ProgressFunc <> nil then + ProgressFunc(Sender, BytesDone, Error^); + libc_close(fsrc); + Exit; + end; + if libc_close(fsrc) <> 0 then + if Result then begin + g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SOURCE_CLOSE), '%m'); + Result := False; + if @ProgressFunc <> nil then + Result := ProgressFunc(Sender, BytesDone, Error^); + if Result then begin + // user has chosen to ignore the error + g_error_free(Error^); + Error^ := nil; + end else Exit; + end; except on E: Exception do begin Result := False; @@ -649,6 +665,8 @@ begin g_error_free(Error^); Error^ := nil; end; + libc_close(fdest); + libc_close(fsrc); end; end; end; @@ -781,39 +799,49 @@ end; (********************************************************************************************************************************) function TLocalTreeEngine.OpenFile(const APath: string; Mode: integer; Error: PPGError): TEngineFileDes; -var m: PChar; +var flags, m: Longint; + r: Integer; begin + Result := nil; + m := 0; case Mode of - omRead: m := 'r'; - omWrite: m := 'w'; - omAppend: m := 'a'; - else m := 'r'; + omRead: flags := ConfDefaultOpenFlagsRead; + omWrite: begin + flags := ConfDefaultOpenFlagsWrite; + m := ConfDefaultFileCreationMask; + end; + omAppend: flags := ConfDefaultOpenFlagsAppend; + else begin + g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_OPEN_FILE), 'Invalid file mode %d', Mode); + Exit; + end; end; - Result := fopen64(PChar(APath), m); - if Result = nil then - g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_OPEN_FILE), '%m'); + + r := open64(PChar(APath), flags, m); + if r < 0 then g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_OPEN_FILE), '%m') + else Result := Pointer(r); end; (********************************************************************************************************************************) function TLocalTreeEngine.ReadFile(const FileDescriptor: TEngineFileDes; Buffer: Pointer; ABlockSize: integer; Error: PPGError): integer; begin - Result := fread(Buffer, 1, ABlockSize, FileDescriptor); - if (Result = 0) and (feof(FileDescriptor) = 0) then + Result := libc_read(LongInt(FileDescriptor), Buffer, ABlockSize); + if Result < 0 then g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_READ_FILE), '%m'); end; (********************************************************************************************************************************) function TLocalTreeEngine.WriteFile(const FileDescriptor: TEngineFileDes; Buffer: Pointer; BytesCount: integer; Error: PPGError): integer; begin - Result := fwrite(Buffer, 1, BytesCount, FileDescriptor); - if Result < BytesCount then + Result := libc_write(LongInt(FileDescriptor), Buffer, BytesCount); + if Result < 0 then g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_WRITE_FILE), '%m'); end; (********************************************************************************************************************************) function TLocalTreeEngine.CloseFile(const FileDescriptor: TEngineFileDes; Error: PPGError): boolean; begin - Result := fclose(FileDescriptor) = 0; + Result := libc_close(LongInt(FileDescriptor)) = 0; if not Result then g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_CLOSE_FILE), '%m'); end; @@ -821,7 +849,7 @@ end; (********************************************************************************************************************************) function TLocalTreeEngine.FileSeek(const FileDescriptor: TEngineFileDes; const AbsoluteOffset: Int64; Error: PPGError): Int64; begin - Result := fseeko64(FileDescriptor, AbsoluteOffset, SEEK_SET); + Result := lseek64(LongInt(FileDescriptor), AbsoluteOffset, SEEK_SET); if Result = -1 then g_set_error(Error, TUXCMD_ERROR, gint(TUXCMD_ERROR_SEEK), '%m'); end; diff --git a/ULibc.pas b/ULibc.pas index edccfc8..3b187be 100644 --- a/ULibc.pas +++ b/ULibc.pas @@ -630,6 +630,23 @@ const SEEK_CUR = 1; SEEK_END = 2; +const + O_RDONLY = 0; + O_WRONLY = 1; + O_RDWR = 2; + O_CREAT = $40; + O_EXCL = $80; + O_NOCTTY = $100; + O_TRUNC = $200; + O_APPEND = $400; + O_NONBLOCK = $800; + O_NDELAY = O_NONBLOCK; + O_SYNC = $1000; + O_DIRECT = $4000; + O_DIRECTORY = $10000; + O_NOFOLLOW = $20000; + O_CLOEXEC = $80000; + const _STAT_VER_LINUX_OLD = 1; _STAT_VER_KERNEL = 1; @@ -872,15 +889,17 @@ function feof(stream: PFILE): Longint; cdecl; external GLIBC_LIB name 'feof'; function ferror(stream: PFILE): Longint; cdecl; external GLIBC_LIB name 'ferror'; function fcntl(fd: Longint; cmd: Longint): Longint; cdecl; varargs; external GLIBC_LIB name 'fcntl'; -function open(const pathname: PChar; flags: Longint): Longint; cdecl; varargs; external GLIBC_LIB name 'open'; -function open64(const pathname: PChar; flags: Longint): Longint; cdecl; varargs; external GLIBC_LIB name 'open64'; +function open(const pathname: PChar; flags: Longint; mode: Longint): Longint; cdecl; varargs; external GLIBC_LIB name 'open'; +function open64(const pathname: PChar; flags: Longint; mode: Longint): Longint; cdecl; varargs; external GLIBC_LIB name 'open64'; function creat(const pathname: PChar; mode: __mode_t): Longint; cdecl; external GLIBC_LIB name 'creat'; function creat64(const pathname: PChar; mode: __mode_t): Longint; cdecl; external GLIBC_LIB name 'creat64'; +function lseek(fd: longint; offset: __off_t; whence: longint):__off_t; cdecl; external GLIBC_LIB name 'lseek'; +function lseek64(fd: longint; offset: __off64_t; whence: longint):__off64_t; cdecl; external GLIBC_LIB name 'lseek64'; -function __read(Handle: Integer; var Buffer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'read'; -function libc_read(Handle: Integer; var Buffer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'read'; -function __write(Handle: Integer; const Buffer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'write'; -function libc_write(Handle: Integer; const Buffer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'write'; +function __read(Handle: Integer; Buffer: Pointer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'read'; +function libc_read(Handle: Integer; Buffer: Pointer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'read'; +function __write(Handle: Integer; Buffer: Pointer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'write'; +function libc_write(Handle: Integer; Buffer: Pointer; Count: size_t): ssize_t; cdecl; external GLIBC_LIB name 'write'; function __close(Handle: Integer): Integer; cdecl; external GLIBC_LIB name 'close'; function libc_close(Handle: Integer): Integer; cdecl; external GLIBC_LIB name 'close'; diff --git a/vfs/uVFSprototypes.pas b/vfs/uVFSprototypes.pas index 9ed0844..6fbe65d 100644 --- a/vfs/uVFSprototypes.pas +++ b/vfs/uVFSprototypes.pas @@ -224,7 +224,7 @@ type // Changes times for the file/directory - mtime and atime are __time_t parameters (glibc) TVFSChangeTimes = function (g:TVFSGlobs; const APath: PChar; mtime, atime: guint32; Error: PPGError): gboolean; cdecl; - + // Performs the copy process from inside of module to local filesystem // (thus sSrcName is a path from inside of module and sDstName is path in the local filesystem where the file should be copied) // Note: if you need to transfer a file between two VFS modules, you need to do it manually - @@ -242,20 +242,19 @@ type TVFSPack = function (g:TVFSGlobs; const sSrcName, sDstName: PChar; CompressionLevel: integer; const Password: PChar; Error: PPGError): gboolean; cdecl; - // TODO: not implemented at all - // This is the set of basic functions which can manipulate with data - // There is a TVFSFileDes object which identifies the processed file (filedescriptor) - // All these functions needs a pointer to an int variable to store the error code + // Low-level open-read-write-close, used for e.g. cross-VFS copy + // An abstract TVFSFileDes object serves as a file descriptor/handle // NOTE: not all modules could support this set of functions due to its design (unable to set a solid block size) TVFSOpenFile = function (g:TVFSGlobs; const APath: PChar; Mode: integer; Error: PPGError): TVFSFileDes; cdecl; // Opens a file or creates new (the values for the Mode parameter are described above) and returns the assigned filedescriptor - TVFSReadFile = function (g:TVFSGlobs; const FileDescriptor: TVFSFileDes; Buffer: Pointer; ABlockSize: integer; Error: PPGError): integer; cdecl; - // Returns number of bytes read; the buffer needs to be allocated by a blocksize (set it by VFSSetBlockSize function) - TVFSWriteFile = function (g:TVFSGlobs; const FileDescriptor: TVFSFileDes; Buffer: Pointer; BytesCount: integer; Error: PPGError): integer; cdecl; - // Returns number of bytes written + TVFSReadFile = function (g:TVFSGlobs; const FileDescriptor: TVFSFileDes; Buffer: Pointer; ABlockSize: guint64; Error: PPGError): gint64; cdecl; + // Returns number of bytes read that may be smaller than requested number of bytes. Returns 0 typically at EOF or -1 in case of an error. + // The buffer needs to be allocated for a blocksize (set it by VFSSetBlockSize function) + TVFSWriteFile = function (g:TVFSGlobs; const FileDescriptor: TVFSFileDes; Buffer: Pointer; BytesCount: guint64; Error: PPGError): gint64; cdecl; + // Returns number of bytes written that may be smaller than requested number of bytes. Returns -1 in case of an error. TVFSCloseFile = function (g:TVFSGlobs; const FileDescriptor: TVFSFileDes; Error: PPGError): gboolean; cdecl; TVFSFileSeek = function (g:TVFSGlobs; const FileDescriptor: TVFSFileDes; const AbsoluteOffset: Int64; Error: PPGError): Int64; cdecl; - // Sets the position in the file from the start and returns real current position + // Sets the position in the file from the start and returns real current position. Returns -1 in case of an error. // Archive-specific routines: -- cgit v1.2.3