From 4806aca183000a472e621388f15cc907f4caeac3 Mon Sep 17 00:00:00 2001
From: Sean Barrett <sean@nothings.org>
Date: Sat, 27 Aug 2016 12:02:26 -0700
Subject: [PATCH] windows fwrite doesn't handle arbitrarily large writes (!);
add a bunch of stuff used by my backup program 'smir';
---
stb.h | 140 +++++++++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 115 insertions(+), 25 deletions(-)
diff --git a/stb.h b/stb.h
index b985b13..18bc37e 100644
--- a/stb.h
+++ b/stb.h
@@ -5189,7 +5189,18 @@ int stb_filewrite(char *filename, void *data, size_t length)
{
FILE *f = stb_fopen(filename, "wb");
if (f) {
- fwrite(data, 1, length, f);
+ unsigned char *data_ptr = (unsigned char *) data;
+ size_t remaining = length;
+ while (remaining > 0) {
+ size_t len2 = remaining > 65536 ? 65536 : remaining;
+ size_t len3 = fwrite(data_ptr, 1, len2, f);
+ if (len2 != len3) {
+ fprintf(stderr, "Failed while writing %s\n", filename);
+ break;
+ }
+ remaining -= len2;
+ data_ptr += len2;
+ }
stb_fclose(f, stb_keep_if_different);
}
return f != NULL;
@@ -5789,6 +5800,19 @@ char *stb_strip_final_slash(char *t)
}
return t;
}
+
+char *stb_strip_final_slash_regardless(char *t)
+{
+ if (t[0]) {
+ char *z = t + strlen(t) - 1;
+ // *z is the last character
+ if (*z == '\\' || *z == '/')
+ *z = 0;
+ if (*z == '\\')
+ *z = '/'; // canonicalize to make sure it matches db
+ }
+ return t;
+}
#endif
//////////////////////////////////////////////////////////////////////////////
@@ -6746,14 +6770,16 @@ typedef struct
char * path; // full path from passed-in root
time_t last_modified;
int num_files;
+ int flag;
} stb_dirtree_dir;
typedef struct
{
char *name; // name relative to path
int dir; // index into dirs[] array
- unsigned long size; // size, max 4GB
+ stb_int64 size; // size, max 4GB
time_t last_modified;
+ int flag;
} stb_dirtree_file;
typedef struct
@@ -6781,6 +6807,14 @@ extern stb_dirtree *stb_dirtree_get_with_file ( char *dir, char *cache_file);
// do a call to stb_dirtree_get() with the same cache file at about the same
// time, but I _think_ it might just work.
+// i needed to build an identical data structure representing the state of
+// a mirrored copy WITHOUT bothering to rescan it (i.e. we're mirroring to
+// it WITHOUT scanning it, e.g. it's over the net), so this requires access
+// to all of the innards.
+extern void stb_dirtree_db_add_dir(stb_dirtree *active, char *path, time_t last);
+extern void stb_dirtree_db_add_file(stb_dirtree *active, char *name, int dir, stb_int64 size, time_t last);
+extern void stb_dirtree_db_read(stb_dirtree *target, char *filename, char *dir);
+extern void stb_dirtree_db_write(stb_dirtree *target, char *filename, char *dir);
#ifdef STB_DEFINE
static void stb__dirtree_add_dir(char *path, time_t last, stb_dirtree *active)
@@ -6792,7 +6826,7 @@ static void stb__dirtree_add_dir(char *path, time_t last, stb_dirtree *active)
stb_arr_push(active->dirs, d);
}
-static void stb__dirtree_add_file(char *name, int dir, unsigned long size, time_t last, stb_dirtree *active)
+static void stb__dirtree_add_file(char *name, int dir, stb_int64 size, time_t last, stb_dirtree *active)
{
stb_dirtree_file f;
f.dir = dir;
@@ -6803,23 +6837,25 @@ static void stb__dirtree_add_file(char *name, int dir, unsigned long size, time_
stb_arr_push(active->files, f);
}
-static char stb__signature[12] = { 's', 'T', 'b', 'D', 'i', 'R', 't', 'R', 'e', 'E', '0', '1' };
+// version 02 supports > 4GB files
+static char stb__signature[12] = { 's', 'T', 'b', 'D', 'i', 'R', 't', 'R', 'e', 'E', '0', '2' };
static void stb__dirtree_save_db(char *filename, stb_dirtree *data, char *root)
{
int i, num_dirs_final=0, num_files_final;
+ char *info = root ? root : "";
int *remap;
FILE *f = fopen(filename, "wb");
if (!f) return;
fwrite(stb__signature, sizeof(stb__signature), 1, f);
- fwrite(root, strlen(root)+1, 1, f);
+ fwrite(info, strlen(info)+1, 1, f);
// need to be slightly tricky and not write out NULLed directories, nor the root
// build remapping table of all dirs we'll be writing out
remap = (int *) malloc(sizeof(remap[0]) * stb_arr_len(data->dirs));
for (i=0; i < stb_arr_len(data->dirs); ++i) {
- if (data->dirs[i].path == NULL || 0==stb_stricmp(data->dirs[i].path, root)) {
+ if (data->dirs[i].path == NULL || (root && 0==stb_stricmp(data->dirs[i].path, root))) {
remap[i] = -1;
} else {
remap[i] = num_dirs_final++;
@@ -6836,14 +6872,14 @@ static void stb__dirtree_save_db(char *filename, stb_dirtree *data, char *root)
num_files_final = 0;
for (i=0; i < stb_arr_len(data->files); ++i)
- if (remap[data->files[i].dir] >= 0)
+ if (remap[data->files[i].dir] >= 0 && data->files[i].name)
++num_files_final;
fwrite(&num_files_final, 4, 1, f);
for (i=0; i < stb_arr_len(data->files); ++i) {
- if (remap[data->files[i].dir] >= 0) {
+ if (remap[data->files[i].dir] >= 0 && data->files[i].name) {
stb_fput_ranged(f, remap[data->files[i].dir], 0, num_dirs_final);
- stb_fput_varlenu(f, data->files[i].size);
+ stb_fput_varlen64(f, data->files[i].size);
fwrite(&data->files[i].last_modified, 4, 1, f);
stb_fput_string(f, data->files[i].name);
}
@@ -6880,7 +6916,7 @@ static void stb__dirtree_load_db(char *filename, stb_dirtree *data, char *dir)
stb_arr_setlen(data->files, n);
for (i=0; i < stb_arr_len(data->files); ++i) {
data->files[i].dir = stb_fget_ranged(f, 0, stb_arr_len(data->dirs));
- data->files[i].size = stb_fget_varlenu(f);
+ data->files[i].size = stb_fget_varlen64(f);
fread(&data->files[i].last_modified, 4, 1, f);
data->files[i].name = stb_fget_string(f, data->string_pool);
if (data->files[i].name == NULL) goto bail;
@@ -6894,6 +6930,7 @@ static void stb__dirtree_load_db(char *filename, stb_dirtree *data, char *dir)
fclose(f);
}
+static int stb__dircount, stb__dircount_mask, stb__showfile;
static void stb__dirtree_scandir(char *path, time_t last_time, stb_dirtree *active)
{
// this is dumb depth first; theoretically it might be faster
@@ -6902,52 +6939,66 @@ static void stb__dirtree_scandir(char *path, time_t last_time, stb_dirtree *acti
int n;
- struct _wfinddata_t c_file;
- #ifdef STB_PTR64
- intptr_t hFile;
- #else
+ struct _wfinddatai64_t c_file;
long hFile;
- #endif
stb__wchar full_path[1024];
int has_slash;
+ if (stb__showfile) printf("<");
has_slash = (path[0] && path[strlen(path)-1] == '/');
if (has_slash)
- swprintf((wchar_t *)full_path, L"%s*", stb__from_utf8(path));
+ swprintf(full_path, L"%s*", stb__from_utf8(path));
else
- swprintf((wchar_t *)full_path, L"%s/*", stb__from_utf8(path));
+ swprintf(full_path, L"%s/*", stb__from_utf8(path));
// it's possible this directory is already present: that means it was in the
// cache, but its parent wasn't... in that case, we're done with it
+ if (stb__showfile) printf("C[%d]", stb_arr_len(active->dirs));
for (n=0; n < stb_arr_len(active->dirs); ++n)
- if (0 == stb_stricmp(active->dirs[n].path, path))
+ if (0 == stb_stricmp(active->dirs[n].path, path)) {
+ if (stb__showfile) printf("D");
return;
+ }
+ if (stb__showfile) printf("E");
// otherwise, we need to add it
stb__dirtree_add_dir(path, last_time, active);
n = stb_arr_lastn(active->dirs);
- if( (hFile = _wfindfirst((const wchar_t *)full_path, &c_file )) != -1L ) {
+ if (stb__showfile) printf("[");
+ if( (hFile = _wfindfirsti64( full_path, &c_file )) != -1L ) {
do {
+ if (stb__showfile) printf(")");
if (c_file.attrib & _A_SUBDIR) {
// ignore subdirectories starting with '.', e.g. "." and ".."
if (c_file.name[0] != '.') {
char *new_path = (char *) full_path;
- char *temp = stb__to_utf8((stb__wchar *)c_file.name);
+ char *temp = stb__to_utf8(c_file.name);
+
if (has_slash)
sprintf(new_path, "%s%s", path, temp);
else
sprintf(new_path, "%s/%s", path, temp);
+
+ if (stb__dircount_mask) {
+ ++stb__dircount;
+ if (!(stb__dircount & stb__dircount_mask)) {
+ printf("%s\r", new_path);
+ }
+ }
+
stb__dirtree_scandir(new_path, c_file.time_write, active);
}
} else {
- char *temp = stb__to_utf8((stb__wchar *)c_file.name);
+ char *temp = stb__to_utf8(c_file.name);
stb__dirtree_add_file(temp, n, c_file.size, c_file.time_write, active);
}
- } while( _wfindnext( hFile, &c_file ) == 0 );
-
+ if (stb__showfile) printf("(");
+ } while( _wfindnexti64( hFile, &c_file ) == 0 );
+ if (stb__showfile) printf("]");
_findclose( hFile );
}
+ if (stb__showfile) printf(">\n");
}
// scan the database and see if it's all valid
@@ -6963,13 +7014,21 @@ static int stb__dirtree_update_db(stb_dirtree *db, stb_dirtree *active)
for (i=0; i < stb_arr_len(db->dirs); ++i) {
struct _stat info;
+ if (stb__dircount_mask) {
+ ++stb__dircount;
+ if (!(stb__dircount & stb__dircount_mask)) {
+ printf(".");
+ }
+ }
if (0 == _stat(db->dirs[i].path, &info)) {
if (info.st_mode & _S_IFDIR) {
// it's still a directory, as expected
- if (info.st_mtime > db->dirs[i].last_modified) {
+ int n = abs(info.st_mtime - db->dirs[i].last_modified);
+ if (n > 1 && n != 3600) { // the 3600 is a hack because sometimes this jumps for no apparent reason, even when no time zone or DST issues are at play
// it's changed! force a rescan
// we don't want to scan it until we've stat()d its
// subdirs, though, so we queue it
+ if (stb__showfile) printf("Changed: %s - %08x:%08x\n", db->dirs[i].path, db->dirs[i].last_modified, info.st_mtime);
stb_arr_push(rescan, i);
// update the last_mod time
db->dirs[i].last_modified = info.st_mtime;
@@ -7047,6 +7106,8 @@ stb_dirtree *stb_dirtree_get_with_file(char *dir, char *cache_file)
if (cache_file != NULL)
stb__dirtree_load_db(cache_file, &db, stripped_dir);
+ else if (stb__showfile)
+ printf("No cache file\n");
active.files = NULL;
active.dirs = NULL;
@@ -7061,6 +7122,9 @@ stb_dirtree *stb_dirtree_get_with_file(char *dir, char *cache_file)
stb__dirtree_scandir(stripped_dir, 0, &active); // no last_modified time available for root
+ if (stb__dircount_mask)
+ printf(" \r");
+
// done with the DB; write it back out if any changes, i.e. either
// 1. any inconsistency found between cached information and actual disk
// or 2. if scanning the root found any new directories--which we detect because
@@ -7089,7 +7153,7 @@ stb_dirtree *stb_dirtree_get_dir(char *dir, char *cache_dir)
stb_sha1(sha, (unsigned char *) dir_lower, strlen(dir_lower));
strcpy(cache_file, cache_dir);
s = cache_file + strlen(cache_file);
- if (s[-1] != '/' && s[-1] != '\\') *s++ = '/';
+ if (s[-1] != '//' && s[-1] != '\\') *s++ = '/';
strcpy(s, "dirtree_");
s += strlen(s);
for (i=0; i < 8; ++i) {
@@ -7123,6 +7187,32 @@ void stb_dirtree_free(stb_dirtree *d)
stb__dirtree_free_raw(d);
free(d);
}
+
+void stb_dirtree_db_add_dir(stb_dirtree *active, char *path, time_t last)
+{
+ stb__dirtree_add_dir(path, last, active);
+}
+
+void stb_dirtree_db_add_file(stb_dirtree *active, char *name, int dir, stb_int64 size, time_t last)
+{
+ stb__dirtree_add_file(name, dir, size, last, active);
+}
+
+void stb_dirtree_db_read(stb_dirtree *target, char *filename, char *dir)
+{
+ char *s = stb_strip_final_slash(strdup(dir));
+ target->dirs = 0;
+ target->files = 0;
+ target->string_pool = 0;
+ stb__dirtree_load_db(filename, target, s);
+ free(s);
+}
+
+void stb_dirtree_db_write(stb_dirtree *target, char *filename, char *dir)
+{
+ stb__dirtree_save_db(filename, target, 0); // don't strip out any directories
+}
+
#endif // STB_DEFINE
#endif // _WIN32