diff --git a/README.md b/README.md index e995116..87b67b5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ libdeflate itself is a library, but the following command-line programs which use this library are also provided: * gzip (or gunzip), a program which mostly behaves like the standard equivalent, - except that it does not yet support reading from standard input and does not + except that it does not yet have good streaming support and therefore does not yet support very large files * benchmark, a program for benchmarking in-memory compression and decompression diff --git a/programs/gzip.c b/programs/gzip.c index 8253c24..81f0c8b 100644 --- a/programs/gzip.c +++ b/programs/gzip.c @@ -382,6 +382,7 @@ decompress_file(struct libdeflate_decompressor *decompressor, const tchar *path, if (ret != 0) goto out_close_in; + /* TODO: need a streaming-friendly solution */ ret = map_file_contents(&in, stbuf.st_size); if (ret != 0) goto out_close_out; @@ -453,8 +454,9 @@ compress_file(struct libdeflate_compressor *compressor, const tchar *path, goto out_close_out; } + /* TODO: need a streaming-friendly solution */ ret = map_file_contents(&in, stbuf.st_size); - if (ret) + if (ret != 0) goto out_close_out; ret = do_compress(compressor, &in, &out); @@ -482,6 +484,7 @@ out_free_newpath: int tmain(int argc, tchar *argv[]) { + tchar *default_file_list[] = { NULL }; struct options options; int opt_char; int i; @@ -547,15 +550,12 @@ tmain(int argc, tchar *argv[]) argc -= toptind; if (argc == 0) { - show_usage(stderr); - return 1; - } - for (i = 0; i < argc; i++) { - if (argv[i][0] == '-' && argv[i][1] == '\0') { - msg("This implementation of gzip does not yet " - "support reading from standard input."); - return 1; - } + argv = default_file_list; + argc = ARRAY_LEN(default_file_list); + } else { + for (i = 0; i < argc; i++) + if (argv[i][0] == '-' && argv[i][1] == '\0') + argv[i] = NULL; } ret = 0; diff --git a/programs/prog_util.c b/programs/prog_util.c index 6a20ce8..68e9ae3 100644 --- a/programs/prog_util.c +++ b/programs/prog_util.c @@ -204,6 +204,7 @@ quote_path(const tchar *path) int xopen_for_read(const tchar *path, bool symlink_ok, struct file_stream *strm) { + strm->mmap_token = NULL; strm->mmap_mem = NULL; if (path == NULL) { @@ -243,6 +244,7 @@ xopen_for_write(const tchar *path, bool overwrite, struct file_stream *strm) { int ret = -1; + strm->mmap_token = NULL; strm->mmap_mem = NULL; if (path == NULL) { @@ -297,14 +299,56 @@ err: return ret; } +/* Read the full contents of a file into memory */ +static int +read_full_contents(struct file_stream *strm) +{ + size_t filled = 0; + size_t capacity = 4096; + char *buf; + int ret; + + buf = xmalloc(capacity); + if (buf == NULL) + return -1; + do { + if (filled == capacity) { + char *newbuf; + + if (capacity == SIZE_MAX) + goto oom; + capacity += MIN(SIZE_MAX - capacity, capacity); + newbuf = realloc(buf, capacity); + if (newbuf == NULL) + goto oom; + buf = newbuf; + } + ret = xread(strm, &buf[filled], capacity - filled); + if (ret < 0) + goto err; + filled += ret; + } while (ret != 0); + + strm->mmap_mem = buf; + strm->mmap_size = filled; + return 0; + +err: + free(buf); + return ret; +oom: + msg("Out of memory! %"TS" is too large to be processed by " + "this program as currently implemented.", strm->name); + ret = -1; + goto err; +} + /* Map the contents of a file into memory */ int map_file_contents(struct file_stream *strm, u64 size) { - if (size == 0) { - strm->mmap_size = 0; - return 0; - } + if (size == 0) /* mmap isn't supported on empty files */ + return read_full_contents(strm); if (size > SIZE_MAX) { msg("%"TS" is too large to be processed by this program", @@ -316,8 +360,11 @@ map_file_contents(struct file_stream *strm, u64 size) (HANDLE)(intptr_t)_get_osfhandle(strm->fd), NULL, PAGE_READONLY, 0, 0, NULL); if (strm->mmap_token == NULL) { + DWORD err = GetLastError(); + if (err == ERROR_BAD_EXE_FORMAT) /* mmap unsupported */ + return read_full_contents(strm); msg("Unable create file mapping for %"TS": Windows error %u", - strm->name, (unsigned int)GetLastError()); + strm->name, (unsigned int)err); return -1; } @@ -333,6 +380,8 @@ map_file_contents(struct file_stream *strm, u64 size) strm->mmap_mem = mmap(NULL, size, PROT_READ, MAP_SHARED, strm->fd, 0); if (strm->mmap_mem == MAP_FAILED) { strm->mmap_mem = NULL; + if (errno == ENODEV) /* mmap isn't supported on this file */ + return read_full_contents(strm); if (errno == ENOMEM) { msg("%"TS" is too large to be processed by this " "program", strm->name); @@ -346,6 +395,7 @@ map_file_contents(struct file_stream *strm, u64 size) #ifdef HAVE_POSIX_MADVISE posix_madvise(strm->mmap_mem, size, POSIX_MADV_SEQUENTIAL); #endif + strm->mmap_token = strm; /* anything that's not NULL */ #endif /* !_WIN32 */ strm->mmap_size = size; @@ -401,23 +451,27 @@ int xclose(struct file_stream *strm) { int ret = 0; - if (strm->fd >= 0 && !strm->is_standard_stream) { + + if (!strm->is_standard_stream) { if (close(strm->fd) != 0) { msg_errno("Error closing %"TS, strm->name); ret = -1; } free(strm->name); - - if (strm->mmap_mem != NULL) { -#ifdef _WIN32 - UnmapViewOfFile(strm->mmap_mem); - CloseHandle((HANDLE)strm->mmap_token); -#else - munmap(strm->mmap_mem, strm->mmap_size); -#endif - strm->mmap_mem = NULL; - } } + + if (strm->mmap_token != NULL) { +#ifdef _WIN32 + UnmapViewOfFile(strm->mmap_mem); + CloseHandle((HANDLE)strm->mmap_token); +#else + munmap(strm->mmap_mem, strm->mmap_size); +#endif + strm->mmap_token = NULL; + } else { + free(strm->mmap_mem); + } + strm->mmap_mem = NULL; strm->fd = -1; strm->name = NULL; return ret;