From 2b162da019150a4f5c0fa1bdbaa315945e1ddd1b Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Tue, 18 Apr 2023 17:39:53 -0600 Subject: [PATCH] update and clean up bfwreaddirplus2db get rid of globals remove commented out code remove unnecessary variables --- .github/workflows/codecov.yml | 2 +- include/bf.h | 1 - include/dbutils.h | 1 - include/template_db.h | 1 + src/bf.c | 2 +- src/bfwreaddirplus2db.c | 1097 +++++++++++++++------------------ src/dbutils.c | 33 - src/external.c | 1 + src/template_db.c | 2 +- test/CMakeLists.txt | 9 +- test/runbfwreaddirplus2db | 163 ++--- test/runtests | 6 +- 12 files changed, 605 insertions(+), 713 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 2a3b571f3..ed101c2aa 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -108,7 +108,7 @@ jobs: run: ctest || true - name: Delete files not reported for coverage - run: find -name "*.gc*" -a \( -name "gendir.*" -o -name "make_testindex.*" -o -name "bffuse.*" -o -name "bfresultfuse.*" -o -name "bfwreaddirplus2db.*" -o -name "dfw.*" -o -name "parallel_rmr.*" -o -name "tsmepoch2time.*" -o -name "tsmtime2epoch.*" -o -path "*/test/*" \) -delete + run: find -name "*.gc*" -a \( -name "gendir.*" -o -name "make_testindex.*" -o -name "bffuse.*" -o -name "bfresultfuse.*" -o -name "dfw.*" -o -name "parallel_rmr.*" -o -name "tsmepoch2time.*" -o -name "tsmtime2epoch.*" -o -path "*/test/*" \) -delete - name: Generate Python Coverage Report run: | diff --git a/include/bf.h b/include/bf.h index 9815e6e72..4e398897f 100644 --- a/include/bf.h +++ b/include/bf.h @@ -186,7 +186,6 @@ struct input { AndOr_t andor; int insertdir; // added for bfwreaddirplus2db int insertfl; // added for bfwreaddirplus2db - int buildinindir; // added to notice when writing index dbs into the input dir int suspectd; // added for bfwreaddirplus2db for how to default suspect directories 0 - not supsect 1 - suspect int suspectfl; // added for bfwreaddirplus2db for how to default suspect file/link 0 - not suspect 1 - suspect refstr_t insuspect; // added for bfwreaddirplus2db input path for suspects file diff --git a/include/dbutils.h b/include/dbutils.h index 17e21004a..e0fe4b86d 100644 --- a/include/dbutils.h +++ b/include/dbutils.h @@ -183,7 +183,6 @@ int insertdbgo_xattrs(struct input *in, struct stat *dir, struct entry_data *dat sll_t *xattr_db_list, struct template_db *xattr_template, const char *topath, const size_t topath_len, sqlite3_stmt *xattrs_res, sqlite3_stmt *xattr_files_res); -int insertdbgor(struct work *pwork, struct entry_data *data, sqlite3_stmt *res); int insertsumdb(sqlite3 *sdb, const char *path, struct work *pwork, struct entry_data *data, struct sum *su); diff --git a/include/template_db.h b/include/template_db.h index 44776b4d7..734faed1b 100644 --- a/include/template_db.h +++ b/include/template_db.h @@ -83,6 +83,7 @@ int init_template_db(struct template_db *tdb); int create_template(struct template_db *tdb, int (*create_tables)(const char *, sqlite3 *, void *), const char *name); int create_xattrs_template(struct template_db *tdb); +int create_dbdb_tables(const char *name, sqlite3 *db, void *args); int create_dbdb_template(struct template_db *tdb); int close_template_db(struct template_db *tdb); diff --git a/src/bf.c b/src/bf.c index f7523e226..0fa7bd7c3 100644 --- a/src/bf.c +++ b/src/bf.c @@ -348,7 +348,7 @@ int parse_cmd_line(int argc, break; case 'A': - INSTALL_INT(&in->suspectmethod, optarg, 1, 4, "-A", &retval); + INSTALL_INT(&in->suspectmethod, optarg, 0, 3, "-A", &retval); break; case 'g': diff --git a/src/bfwreaddirplus2db.c b/src/bfwreaddirplus2db.c index 13af0beec..113080e8a 100644 --- a/src/bfwreaddirplus2db.c +++ b/src/bfwreaddirplus2db.c @@ -62,21 +62,15 @@ OF SUCH DAMAGE. -#include #include #include #include -#include #include #include #include #include #include #include -#include - -#include -#include #include "QueuePerThreadPool.h" #include "bf.h" @@ -85,206 +79,193 @@ OF SUCH DAMAGE. #include "utils.h" #include "xattrs.h" -// global variable to hold per thread state goes here -struct { - FILE* outfd[MAXPTHREAD]; - sqlite3* outdbd[MAXPTHREAD]; -} gts = {}; +struct ThreadArgs { + pthread_mutex_t mutex; + FILE *file; + sqlite3 *db; + sqlite3_stmt *res; +}; -pthread_mutex_t outdb_mutex[MAXPTHREAD]; -pthread_mutex_t outfile_mutex[MAXPTHREAD]; -sqlite3_stmt *global_res[MAXPTHREAD]; +struct PoolArgs { + struct input in; + ino_t glsuspectflmin; + ino_t glsuspectflmax; + ino_t glsuspectdmin; + ino_t glsuspectdmax; + int gltodirmode; -long long int glsuspectflmin; -long long int glsuspectflmax; -long long int glsuspectdmin; -long long int glsuspectdmax; + trie_t *dirs; + trie_t *fls; -int gltodirmode; + struct ThreadArgs *ta; +}; -/* a triell for directories and one for files and links */ -trie_t *headd; -trie_t *headfl; +enum DFL { + DFL_DIR, + DFL_FL, +}; /* search the trie either the directory trie type 0 or the filelink type 1 */ -int searchmyll(long long int lull, int lutype) { - char lut[256]; - int ret; - - ret=0; - if (lutype==0) { - //printf("in searchmyll before dirheadd %lld %d ret %d\n",lull, lutype,ret); - if (headd==NULL) return 0; - if (lull < glsuspectdmin) return 0; - if (lull > glsuspectdmax) return 0; - const size_t len = SNPRINTF(lut, 256, "%lld", lull); - ret = trie_search(headd, lut, len); - //printf("in searchmyll search dir %lld %d ret %d lut %s\n",lull, lutype,ret,lut); - // if (ret==1) deletionll(&headd,lut); this is not thread safe - } - if (lutype==1) { - if (headfl==NULL) return 0; - if (lull < glsuspectflmin) return 0; - if (lull > glsuspectflmax) return 0; - const size_t len = SNPRINTF(lut, 256, "%lld", lull); - ret = trie_search(headfl, lut, len); - //printf("in searchmyll search fl %lld %d ret %d lut %s\n",lull, lutype,ret,lut); - // if (ret==1) deletionll(&headfl,lut); this is not thread safe - } - //printf("in searchmyll %lld %d ret %d\n",lull, lutype,ret); - return(ret); +static int searchmyll(struct PoolArgs *pa, ino_t inode, const enum DFL type) { + trie_t *dst = NULL; + if (type == DFL_DIR) { + if (inode < pa->glsuspectdmin) return 0; + if (inode > pa->glsuspectdmax) return 0; + dst = pa->dirs; + } + else if (type == DFL_FL) { + if (inode < pa->glsuspectflmin) return 0; + if (inode > pa->glsuspectflmax) return 0; + dst = pa->fls; + } + + char str[256]; + const size_t len = SNPRINTF(str, sizeof(str), "%" STAT_ino, inode); + return trie_search(dst, str, len); } -static int create_tables(const char * name, sqlite3 * db, void * args) { +static int create_readdirplus_tables(const char *name, sqlite3 *db, void *args) { (void) args; - if ((create_table_wrapper(name, db, ENTRIES, ENTRIES_CREATE) != SQLITE_OK) || - (create_table_wrapper(name, db, SUMMARY, SUMMARY_CREATE) != SQLITE_OK) || - (create_table_wrapper(name, db, PENTRIES_ROLLUP, PENTRIES_ROLLUP_CREATE) != SQLITE_OK) || - (create_table_wrapper(name, db, PENTRIES, PENTRIES_CREATE) != SQLITE_OK) || - (create_table_wrapper(name, db, "vssqldir", vssqldir) != SQLITE_OK) || - (create_table_wrapper(name, db, "vssqluser", vssqluser) != SQLITE_OK) || - (create_table_wrapper(name, db, "vssqlgroup", vssqlgroup) != SQLITE_OK)) { + if (create_table_wrapper(name, db, READDIRPLUS, READDIRPLUS_CREATE) != SQLITE_OK) { return -1; } return 0; } -static int create_readdirplus_tables(const char * name, sqlite3 * db, void * args) { - (void) args; - - if (create_table_wrapper(name, db, READDIRPLUS, READDIRPLUS_CREATE) != SQLITE_OK) { - return -1; +static int insertdbgor(struct work *pwork, struct entry_data *ed, sqlite3_stmt *res) +{ + int rc = 0; + char *zname = sqlite3_mprintf("%q", pwork->name); + char *ztype = sqlite3_mprintf("%c", ed->type); + int error = sqlite3_bind_text(res, 1, zname, -1, SQLITE_TRANSIENT); + if (error != SQLITE_OK) { + fprintf(stderr, "SQL insertdbgor bind name: %s error %d err %s\n", + pwork->name, error, sqlite3_errstr(error)); + rc = 1; + goto cleanup; + } + sqlite3_bind_text(res, 2, ztype, -1, SQLITE_TRANSIENT); + sqlite3_bind_int64(res, 3, ed->statuso.st_ino); + sqlite3_bind_int64(res, 4, pwork->pinode); + sqlite3_bind_int64(res, 5, ed->suspect); + + error = sqlite3_step(res); + if (error != SQLITE_DONE) { + fprintf(stderr, "SQL insertdbgor step: %s error %d err %s\n", + pwork->name, error, sqlite3_errstr(error)); + rc = 1; } - return 0; + cleanup: + sqlite3_free(zname); + sqlite3_free(ztype); + sqlite3_reset(res); + sqlite3_clear_bindings(res); + + return rc; } -int reprocessdir(struct input *in, void * passv, DIR *dir) -{ +static int reprocessdir(struct input *in, void *passv, DIR *dir) { struct work *passmywork = passv; + struct entry_data ed; - struct work qwork; - struct entry_data qwork_ed; - //DIR *dir; - struct dirent *entry = NULL; - sqlite3 *db = NULL; - char *records = NULL; - char lpatho[MAXPATH]; - struct sum summary; - sqlite3_stmt *res = NULL; - char dbpath[MAXPATH]; - int transcnt; - int loop; + memset(&ed, 0, sizeof(ed)); if (lstat(passmywork->name, &ed.statuso) != 0) { return 1; } - // rewind the directory - rewinddir(dir); - - //printf(" in reprocessdir rebuilding gufi for %s\n",passmywork->name); - /* need to fill this in for the directory as we dont need to do this unless we are making a new gufi db */ - bzero(ed.linkname,sizeof(ed.linkname)); - ed.type = 'd'; - xattrs_setup(&ed.xattrs); if (in->external_enabled) { - xattrs_get(passmywork->name, &ed.xattrs); + xattrs_setup(&ed.xattrs); + xattrs_get(passmywork->name, &ed.xattrs); } - - //open the gufi db for this directory into the parking lot directory the name as the inode of the dir - SNPRINTF(dbpath,MAXPATH,"%s/%"STAT_ino"",in->nameto.data,ed.statuso.st_ino); - if (in->buildinindir == 1) { - SNPRINTF(dbpath,MAXPATH,"%s/%s",passmywork->name,DBNAME); + /* open the gufi db for this directory into the parking lot directory the name as the inode of the dir */ + char dbpath[MAXPATH]; + if (in->buildindex == 1) { + SNPRINTF(dbpath, MAXPATH, "%s/%s", passmywork->name, DBNAME); } else { - SNPRINTF(dbpath,MAXPATH,"%s/%"STAT_ino"",in->nameto.data,ed.statuso.st_ino); + SNPRINTF(dbpath, MAXPATH, "%s/%" STAT_ino, in->nameto.data, ed.statuso.st_ino); } - /* if we are building a gufi in the src tree and the suspect mode is not zero then we need to wipe it out first */ - if (in->buildinindir == 1) { + /* + * if we are building a gufi in the src tree and the suspect mode + * is not zero then we need to wipe it out first + */ + if (in->buildindex == 1) { if (in->suspectmethod > 0) { - //unlink(dbpath); - truncate(dbpath,0); + truncate(dbpath, 0); } } - if (!(db = opendb(dbpath, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 1, 1 - , create_tables, NULL - #if defined(DEBUG) && defined(PER_THREAD_STATS) - , NULL, NULL - , NULL, NULL - #endif - ))) { - return -1; + + sqlite3 *db = opendb(dbpath, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 1, 1 + , create_dbdb_tables, NULL + #if defined(DEBUG) && defined(PER_THREAD_STATS) + , NULL, NULL + , NULL, NULL + #endif + ); + if (!db) { + return -1; } - res=insertdbprep(db, ENTRIES_INSERT); + + sqlite3_stmt *res = insertdbprep(db, ENTRIES_INSERT); startdb(db); - records=malloc(MAXRECS); - bzero(records,MAXRECS); + struct sum summary; zeroit(&summary); - // loop over dirents, if link push it on the queue, if file or link - // print it, fill up qwork structure for each - transcnt = 0; - loop=1; - while (loop == 1) { - - /* get the next dirent */ - if (!(entry = readdir(dir))) break; - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) - continue; - if (in->buildinindir == 1) { - if (strcmp(entry->d_name, DBNAME) == 0) - continue; + /* rewind the directory */ + rewinddir(dir); + + /* + * loop over dirents, if link push it on the queue, if file or + * link print it, fill up qwork structure for each + */ + struct dirent *entry = NULL; + while ((entry = readdir(dir))) { + const size_t len = strlen(entry->d_name); + + if (((len == 1) && !strncmp(entry->d_name, ".", 1)) || + ((len == 2) && !strncmp(entry->d_name, "..", 2))) + continue; + + if (in->buildindex == 1) { + if ((len == DBNAME_LEN) && !strncmp(entry->d_name, DBNAME, DBNAME_LEN)) + continue; } - //printf("reprocessdir: dir %s entry %s\n",passmywork->name,entry->d_name); + struct work qwork; + struct entry_data qwork_ed; memset(&qwork, 0, sizeof(qwork)); memset(&qwork_ed, 0, sizeof(qwork_ed)); - qwork.pinode=ed.statuso.st_ino; - //if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) - // continue; - SNPRINTF(qwork.name,MAXPATH,"%s/%s", passmywork->name, entry->d_name); + qwork.pinode = ed.statuso.st_ino; + + qwork.name_len = SNPRINTF(qwork.name, MAXPATH, "%s/%s", passmywork->name, entry->d_name); + qwork.basename_len = len; + lstat(qwork.name, &qwork_ed.statuso); - /* qwork.xattrs_len = 0; */ xattrs_setup(&qwork_ed.xattrs); if (in->external_enabled) { - xattrs_get(qwork.name, &qwork_ed.xattrs); + xattrs_get(qwork.name, &qwork_ed.xattrs); } - if (S_ISDIR(qwork_ed.statuso.st_mode) ) { - // this is how the parent gets passed on - //qwork.pinode=ed.statuso.st_ino; - // there is no work to do for a directory here - we are processing files and links of this dir into a gufi db - xattrs_cleanup(&qwork_ed.xattrs); - continue; - } else if (S_ISLNK(qwork_ed.statuso.st_mode) ) { - // its a link so get the linkname - bzero(lpatho,sizeof(lpatho)); - readlink(qwork.name,lpatho,MAXPATH); - SNPRINTF(qwork_ed.linkname,MAXPATH,"%s",lpatho); - qwork_ed.type = 'l'; - //sprintf(qwork.linkname,"%s/%s",passmywork->name,lpatho); - sumit(&summary,&qwork_ed); - insertdbgo(&qwork,&qwork_ed,res); - transcnt++; - if (transcnt > 100000) { - stopdb(db); - startdb(db); - transcnt=0; - } - } else if (S_ISREG(qwork_ed.statuso.st_mode) ) { - qwork_ed.type = 'f'; - sumit(&summary,&qwork_ed); - insertdbgo(&qwork,&qwork_ed,res); - transcnt++; - if (transcnt > 100000) { - stopdb(db); - startdb(db); - transcnt=0; + + /* + * there is no work to do for a directory here - we are + * processing files and links of this dir into a gufi db + */ + if (!S_ISDIR(qwork_ed.statuso.st_mode)) { + if (S_ISLNK(qwork_ed.statuso.st_mode)) { + qwork_ed.type = 'l'; + readlink(qwork.name, qwork_ed.linkname, MAXPATH); + } else if (S_ISREG(qwork_ed.statuso.st_mode)) { + qwork_ed.type = 'f'; } + + sumit(&summary, &qwork_ed); + insertdbgo(&qwork, &qwork_ed, res); } xattrs_cleanup(&qwork_ed.xattrs); @@ -293,529 +274,471 @@ int reprocessdir(struct input *in, void * passv, DIR *dir) stopdb(db); insertdbfin(res); - // this i believe has to be after we close off the entries transaction + /* this i believe has to be after we close off the entries transaction */ insertsumdb(db, passmywork->name, passmywork, &ed, &summary); xattrs_cleanup(&ed.xattrs); closedb(db); chown(dbpath, ed.statuso.st_uid, ed.statuso.st_gid); - chmod(dbpath, ed.statuso.st_mode | S_IRUSR); - free(records); + chmod(dbpath, (ed.statuso.st_mode & ~(S_IXUSR | S_IXGRP | S_IXOTH)) | S_IRUSR); return 0; +} +static int stripe_index(struct input *in, const ino_t inode, const int id) { + if (in->stride > 0) { + return (inode / in->stride) % in->maxthreads; /* striping inodes */ + } + return id; } -// This becomes an argument to thpool_add_work(), so it must return void, -// instead of void*. -static int processdir(QPTPool_t * ctx, const size_t id, void * data, void * args) -{ +static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) { struct work *passmywork = data; - struct entry_data ed; - struct work qwork; - struct entry_data qwork_ed; - DIR *dir; - struct dirent *entry; - char dbpath[MAXPATH]; - char sortf[MAXPATH]; - int transcnt; - int wentry; - int todb = 0; - int tooutfile; - int lookup; - struct stat st; - int rc; - int locsuspecttime; - struct stat sst; - - struct input *in = (struct input *) args; - - // open directory - if (!(dir = opendir(passmywork->name))) - goto out_free; // return NULL; - - if (lstat(passmywork->name, &ed.statuso) != 0) { + struct PoolArgs *pa = (struct PoolArgs *) args; + struct input *in = &pa->in; + + DIR *dir = opendir(passmywork->name); + if (!dir) { + const int err = errno; + fprintf(stderr, "couldn't open dir '%s': %s\n", + passmywork->name, strerror(err)); goto out_free; } - if (!(entry = readdir(dir))) - goto out_dir; // return NULL; + struct entry_data ed; + memset(&ed, 0, sizeof(ed)); + if (lstat(passmywork->name, &ed.statuso) != 0) { + const int err = errno; + fprintf(stderr, "couldn't stat dir '%s': %s\n", + passmywork->name, strerror(err)); + goto out_dir; + } ed.type = 'd'; - ed.suspect=in->suspectd; - /* if we are putting the gufi index into the source tree we can modify the suspecttime to be the mtime of the gufi db */ - /* this way we will just be looking at dirs or files that have changed since the gufi db was last updated */ - locsuspecttime=in->suspecttime; - if (in->buildinindir == 1) { - locsuspecttime=0; - SNPRINTF(dbpath,MAXPATH,"%s/%s",passmywork->name,DBNAME); - rc=lstat(dbpath,&sst); - if (rc == 0) { - locsuspecttime=sst.st_mtime; - } else { - ed.suspect=1; - } - //printf("in processdir setting suspecttime: set to %d rc from stat %d\n",locsuspecttime,rc); + ed.suspect = in->suspectd; + + /* + * if we are putting the gufi index into the source tree we can + * modify the suspecttime to be the mtime of the gufi db + * + * this way we will just be looking at dirs or files that have + * changed since the gufi db was last updated + */ + int locsuspecttime = in->suspecttime; + if (in->buildindex == 1) { + locsuspecttime = 0; /* all timestamps are suspect */ + char dbpath[MAXPATH]; + SNPRINTF(dbpath, MAXPATH, "%s/%s", passmywork->name, DBNAME); + struct stat st; + if (lstat(dbpath, &st) == 0) { + locsuspecttime = st.st_mtime; + } else { + ed.suspect = 1; + } } - /* if we are not looking for suspect directories we should just put the directory at the top of all the dirents */ + /* + * if we are not looking for suspect directories we should + * just put the directory at the top of all the dirents + */ if (in->output == OUTDB) { - if (in->insertdir > 0) { - if (in->suspectmethod == 0) { - todb=id; - if (in->stride > 0) { - todb=(ed.statuso.st_ino/in->stride)%in->maxthreads; //striping inodes - //******** start a lock - pthread_mutex_lock(&outdb_mutex[todb]); - } - //printf("in processdirs in.outdb=%d in.insertdir=%d id=%d in.insertfl=%d\n",in.outdb,in.insertdir,id,in.insertfl); - startdb(gts.outdbd[todb]); - insertdbgor(passmywork,&ed,global_res[todb]); - if (in->stride > 0) { - stopdb(gts.outdbd[todb]); //striping inodes - //***** drop a lock - pthread_mutex_unlock(&outdb_mutex[todb]); - } - } - if (in->suspectmethod == 1) { /* look up inode in trie to see if this is a suspect dir */ - lookup=0; - lookup=searchmyll(ed.statuso.st_ino,0); - if (lookup == 1) ed.suspect=1; /* set the directory suspect flag on so we will mark it in output */ - } - if (in->suspectmethod > 1) { - /* ????? we would add a stat call on the directory here and compare mtime and ctime with the last run time provided */ - /* and mark the dir suspect if mtime or ctime are >= provided last run time */ - // needed to fill in passmywork status structure - ed.statuso.st_ctime=0; - ed.statuso.st_mtime=0; - lstat(passmywork->name,&ed.statuso); - if (ed.statuso.st_ctime >= locsuspecttime) ed.suspect=1; - if (ed.statuso.st_mtime >= locsuspecttime) ed.suspect=1; - } - } - } + if (in->insertdir > 0) { + if (in->suspectmethod == 0) { + const int todb = stripe_index(in, ed.statuso.st_ino, id); + struct ThreadArgs *ta = &pa->ta[todb]; + + if (in->stride > 0) { + pthread_mutex_lock(&ta->mutex); + } - if (in->output == OUTFILE) { - tooutfile=id; - if (in->stride > 0) { - tooutfile=(ed.statuso.st_ino/in->stride)%in->maxthreads; //striping inodes - //******** start a lock - pthread_mutex_lock(&outfile_mutex[todb]); - } - //fprintf(stderr,"threadd %d inode %lld file %d\n",id,passmywork->statuso.st_ino,tooutfile); - /* only directories are here so sortf is set to the directory full pathname */ - SNPRINTF(sortf,MAXPATH,"%s",passmywork->name); - fprintf(gts.outfd[tooutfile], "%s%c%"STAT_ino"%c%lld%c%c%c%s%c\n", - passmywork->name, in->delim, - ed.statuso.st_ino, in->delim, - passmywork->pinode, in->delim, - ed.type, in->delim, - sortf, in->delim); - if (in->stride > 0) { - pthread_mutex_unlock(&outfile_mutex[todb]); - } + startdb(ta->db); + insertdbgor(passmywork, &ed, ta->res); + + if (in->stride > 0) { + stopdb(ta->db); + pthread_mutex_unlock(&ta->mutex); + } + } + else if (in->suspectmethod == 1) { /* look up inode in trie to see if this is a suspect dir */ + ed.suspect = searchmyll(pa, ed.statuso.st_ino, DFL_DIR); /* set the directory suspect flag on so we will mark it in output */ + } + else if (in->suspectmethod > 1) { + /* mark the dir suspect if mtime or ctime are >= provided last run time */ + lstat(passmywork->name, &ed.statuso); + if (ed.statuso.st_ctime >= locsuspecttime) ed.suspect = 1; + if (ed.statuso.st_mtime >= locsuspecttime) ed.suspect = 1; + } + } } -/* - if (in.printing > 0 || in.printdir > 0) { - printits(passmywork,id); + else if (in->output == OUTFILE) { + const int tooutfile = stripe_index(in, ed.statuso.st_ino, id); + struct ThreadArgs *ta = &pa->ta[tooutfile]; + + if (in->stride > 0) { + pthread_mutex_lock(&ta->mutex); + } + + /* only directories are here so sortf is set to the directory full pathname */ + fprintf(ta->file, "%s%c%" STAT_ino "%c%lld%c%c%c%s%c\n", + passmywork->name, in->delim, + ed.statuso.st_ino, in->delim, + passmywork->pinode, in->delim, + ed.type, in->delim, + passmywork->name, in->delim); + + if (in->stride > 0) { + pthread_mutex_unlock(&ta->mutex); + } } -*/ - // loop over dirents, if dir push it on the queue, if file or link - // print it, fill up qwork structure for each - transcnt=0; - do { - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) - continue; - if (in->buildinindir == 1) { - if (strcmp(entry->d_name, DBNAME) == 0) - continue; + + /* + * loop over dirents, if dir push it on the queue, if file or link + * print it, fill up qwork structure for each + */ + struct dirent *entry = NULL; + while ((entry = readdir(dir))) { + const size_t len = strlen(entry->d_name); + + if (((len == 1) && !strncmp(entry->d_name, ".", 1)) || + ((len == 2) && !strncmp(entry->d_name, "..", 2))) + continue; + + if (in->buildindex == 1) { + if ((len == DBNAME_LEN) && !strncmp(entry->d_name, DBNAME, DBNAME_LEN)) + continue; } - bzero(&qwork,sizeof(qwork)); - SNPRINTF(qwork.name,MAXPATH,"%s/%s", passmywork->name, entry->d_name); - qwork.pinode=ed.statuso.st_ino; - qwork_ed.statuso.st_ino=entry->d_ino; - qwork_ed.suspect=in->suspectfl; - wentry=0; - //if (S_ISDIR(qwork.statuso.st_mode) ) { - if (entry->d_type==DT_DIR) { - if (!access(qwork.name, R_OK | X_OK)) { - qwork_ed.type = 'd'; - struct work * work = malloc(sizeof(struct work)); - *work = qwork; - QPTPool_enqueue(ctx, id, processdir, work); - } - } else if (entry->d_type==DT_LNK) { + + struct work qwork; + struct entry_data qwork_ed; + memset(&qwork, 0, sizeof(qwork)); + memset(&qwork_ed, 0, sizeof(qwork_ed)); + + SNPRINTF(qwork.name, MAXPATH, "%s/%s", passmywork->name, entry->d_name); + qwork.pinode = ed.statuso.st_ino; + qwork_ed.statuso.st_ino = entry->d_ino; + + if (entry->d_type == DT_DIR) { + struct work *work = calloc(1, sizeof(struct work)); + memcpy(work, &qwork, sizeof(qwork)); + QPTPool_enqueue(ctx, id, processdir, work); + continue; + } + else if (entry->d_type == DT_LNK) { qwork_ed.type = 'l'; - wentry=1; - } else if (entry->d_type==DT_REG) { + } + else if (entry->d_type == DT_REG) { qwork_ed.type = 'f'; - wentry=1; } - if (wentry==1) { -/* - if (in.printing > 0) { - printits(&qwork,id); - } -*/ - /* if suspect method is not zero then we can insert files and links, if not we dont care about files and links in db */ - if (in->suspectmethod == 0) { + + /* + * if suspect method is not zero then we can insert files and links + * if not we dont care about files and links in db + */ + if (in->suspectmethod) { if (in->output == OUTDB) { - if (in->insertfl > 0) { - todb=id; + if (in->insertfl > 0) { + const int todb = stripe_index(in, ed.statuso.st_ino, id); + struct ThreadArgs *ta = &pa->ta[todb]; + + if (in->stride > 0) { + pthread_mutex_lock(&ta->mutex); + startdb(ta->db); + } + + insertdbgor(&qwork, &qwork_ed, ta->res); + + if (in->stride > 0) { + stopdb(ta->db); + pthread_mutex_unlock(&ta->mutex); + } + } + } + + if (ed.suspect == 0) { /* if suspect dir just skip looking any further */ + if ((in->suspectmethod == 1) || + (in->suspectmethod == 2)) { /* if method 1 or 2 we look up the inode in trie and mark dir suspect or not */ + ed.suspect = searchmyll(pa, qwork_ed.statuso.st_ino, DFL_FL); /* set the directory suspect flag on so we will mark it in output */ + } + if (in->suspectmethod > 2) { + /* ???? we would stat the file/link and if ctime or mtime is >= provided last run time mark dir suspect */ + struct stat st; + st.st_ctime = 0; + st.st_mtime = 0; + lstat(qwork.name, &st); + if (st.st_ctime >= locsuspecttime) ed.suspect = 1; + if (st.st_mtime >= locsuspecttime) ed.suspect = 1; + } + } + + if (in->output == OUTFILE) { + const int tooutfile = stripe_index(in, ed.statuso.st_ino, id); + struct ThreadArgs *ta = &pa->ta[tooutfile]; + if (in->stride > 0) { - todb=(qwork_ed.statuso.st_ino/in->stride)%in->maxthreads; //striping inodes - //************** start a lock - pthread_mutex_lock(&outdb_mutex[todb]); - startdb(gts.outdbd[todb]); //striping inodes + pthread_mutex_lock(&ta->mutex); } - insertdbgor(&qwork,&qwork_ed,global_res[todb]); + + /* + * since this is a file or link, we need the path to + * the file or link without the name as the sortf + */ + fprintf(ta->file, "%s%c%" STAT_ino "%c%lld%c%c%c%s%c\n", + qwork.name, in->delim, + qwork_ed.statuso.st_ino, in->delim, + qwork.pinode, in->delim, + qwork_ed.type, in->delim, + passmywork->name, in->delim); + if (in->stride > 0) { - stopdb(gts.outdbd[todb]); //striping inodes - //************** drop a lock - pthread_mutex_unlock(&outdb_mutex[todb]); - } else { - transcnt++; - if (transcnt > 100000) { - stopdb(gts.outdbd[todb]); - startdb(gts.outdbd[todb]); - transcnt=0; - } + pthread_mutex_unlock(&ta->mutex); } - } - } - } - if (ed.suspect == 0) { /* if suspect dir just skip looking any further */ - if ((in->suspectmethod == 1) || (in->suspectmethod == 2)) { /* if method 1 or 2 we look up the inode in trie and mark dir suspect or not */ - lookup=0; - lookup=searchmyll(qwork_ed.statuso.st_ino,1); - if (lookup == 1) { - ed.suspect=1; /* set the directory suspect flag on so we will mark it in output */ - //printf("######### found a file suspect %s %lld\n",qwork.name,qwork.statuso.st_ino); - } - } - if (in->suspectmethod > 2) { - /* ???? we would stat the file/link and if ctime or mtime is >= provided last run time mark dir suspect */ - st.st_ctime=0; - st.st_mtime=0; - lstat(qwork.name,&st); - if (st.st_ctime >= locsuspecttime) ed.suspect=1; - if (st.st_mtime >= locsuspecttime) ed.suspect=1; } - } - if (in->output == OUTFILE) { - tooutfile=id; - if (in->stride > 0) { - tooutfile=(qwork_ed.statuso.st_ino/in->stride)%in->maxthreads; //striping inodes - //******** start a lock - pthread_mutex_lock(&outfile_mutex[todb]); - } - //fprintf(stderr,"threadf %d inode %lld file %d\n",id,qwork.statuso.st_ino,tooutfile); - /* since this is a file or link, we need the path to the file or link without the name as the sortf */ - SNPRINTF(sortf,MAXPATH,"%s",passmywork->name); - fprintf(gts.outfd[tooutfile], "%s%c%"STAT_ino"%c%lld%c%c%c%s%c\n", - qwork.name, in->delim, - qwork_ed.statuso.st_ino, in->delim, - qwork.pinode, in->delim, - qwork_ed.type, in->delim, - sortf, in->delim); - if (in->stride > 0) { - pthread_mutex_unlock(&outfile_mutex[todb]); - } - } } - } while ((entry = (readdir(dir)))); + } - /* if we are not looking for suspect directories we should just put the directory at the top of all the dirents */ + /* + * if we are not looking for suspect directories we should just + * put the directory at the top of all the dirents + */ if (in->suspectmethod > 0) { - if (in->output == OUTFILE) { - if (in->insertdir > 0) { - todb=id; - if (in->stride > 0) { - todb=(ed.statuso.st_ino/in->stride)%in->maxthreads; //striping inodes - //******** start a lock - pthread_mutex_lock(&outdb_mutex[todb]); - } - startdb(gts.outdbd[todb]); - //printf("in processdirs in.outdb=%d in.insertdir=%d id=%d in.insertfl=%d\n",in.outdb,in.insertdir,id,in.insertfl); - insertdbgor(passmywork,&ed,global_res[todb]); - if (in->stride > 0) { - stopdb(gts.outdbd[todb]); //striping inodes - //***** drop a lock - pthread_mutex_unlock(&outdb_mutex[todb]); - } - } - } + if (in->output == OUTDB) { + if (in->insertdir > 0) { + const int todb = stripe_index(in, ed.statuso.st_ino, id); + struct ThreadArgs *ta = &pa->ta[todb]; + + if (in->stride > 0) { + pthread_mutex_lock(&ta->mutex); + } + + startdb(ta->db); + insertdbgor(passmywork, &ed, ta->res); + + if (in->stride > 0) { + stopdb(ta->db); + pthread_mutex_unlock(&ta->mutex); + } + } + } } if (in->output == OUTDB) { - if (in->stride == 0) { - todb=id; - stopdb(gts.outdbd[todb]); - } + if (in->insertdir > 0) { + if (in->stride == 0) { + stopdb(pa->ta[id].db); + } + } } - if (ed.suspect==1) { - if (gltodirmode==1) { - /* we may not have stat on the directory we may be told its suspect somehow not stating it */ - /* lstat(passmywork->name,&ed.statuso); */ - rc=reprocessdir(in,passmywork,dir); - if (rc !=0) fprintf(stderr,"problem producing gufi db for suspect directory\n"); - } + if (ed.suspect == 1) { + if (pa->gltodirmode == 1) { + /* we may not have stat on the directory we may be told its suspect somehow not stating it */ + if (reprocessdir(in, passmywork, dir)) { + fprintf(stderr, "problem producing gufi db for suspect directory\n"); + } + } } out_dir: - // close dir closedir(dir); out_free: - - // free the queue entry - this has to be here or there will be a leak free(passmywork); return 0; } -int processinit(struct input *in, QPTPool_t * ctx) { - - struct work * mywork = malloc(sizeof(struct work)); - size_t i; - char outdbn[MAXPATH]; - FILE *isf = NULL; +static int processinit(struct PoolArgs *pa, QPTPool_t *ctx) { char incsuspect[24]; - char incsuspecttype[2]; - long long int testll; - long long int cntd; - long long int cntfl; - char outfn[MAXPATH]; - - if (in->suspectfile > 0) { - if( (isf = fopen(in->insuspect.data, "r")) == NULL) - { - fprintf(stderr,"Cant open input suspect file %s\n",in->insuspect.data); - exit(1); - } - cntfl=0; - cntd=0; - /* set up triell for directories and one for files and links */ - headd = trie_alloc(); - headfl = trie_alloc(); - while (fscanf(isf,"%s %s",incsuspect, incsuspecttype)!= EOF) { - //printf("insuspect |%s| |%s|\n",incsuspect, incsuspecttype ); - testll=atoll(incsuspect); - if (!strncmp(incsuspecttype,"f",1)) { - if (cntfl==0) { - glsuspectflmin=testll; - glsuspectflmax=testll;; - } else { - if (testll < glsuspectflmin) glsuspectflmin=testll; - if (testll > glsuspectflmax) glsuspectflmax=testll; - } - //printf("insuspect %s %s %lld %lld\n",incsuspect, incsuspecttype,glsuspectflmin,glsuspectflmax ); - trie_insert(headfl, incsuspect,1); - cntfl++; - } - if (!strncmp(incsuspecttype,"l",1)) { - if (cntfl==0) { - glsuspectflmin=testll; - glsuspectflmax=testll;; - } else { - if (testll < glsuspectflmin) glsuspectflmin=testll; - if (testll > glsuspectflmax) glsuspectflmax=testll; + char incsuspecttype; + char outname[MAXPATH]; + + if (pa->in.suspectfile > 0) { + FILE *isf = fopen(pa->in.insuspect.data, "r"); + if(!isf) { + fprintf(stderr, "Cant open input suspect file %s\n", pa->in.insuspect.data); + return 1; + } + + long long int cntfl = 0; + long long int cntd = 0; + + /* set up triell for directories and one for files and links */ + pa->dirs = trie_alloc(); + pa->fls = trie_alloc(); + + while (fscanf(isf,"%s %c", incsuspect, &incsuspecttype) != EOF) { + ino_t testll = 0; + if (sscanf(incsuspect, "%" STAT_ino, &testll) != 1) { + continue; } - //printf("insuspect %s %s %lld %lld\n",incsuspect, incsuspecttype,glsuspectflmin,glsuspectflmax ); - trie_insert(headfl, incsuspect, 1); - cntfl++; - } - if (!strncmp(incsuspecttype,"d",1)) { - if (cntd==0) { - glsuspectdmin=testll; - glsuspectdmax=testll;; - } else { - if (testll < glsuspectdmin) glsuspectdmin=testll; - if (testll > glsuspectdmax) glsuspectdmax=testll; + + switch (incsuspecttype) { + case 'd': + if (cntd==0) { + pa->glsuspectdmin = testll; + pa->glsuspectdmax = testll; + } else { + if (testll < pa->glsuspectdmin) pa->glsuspectdmin = testll; + if (testll > pa->glsuspectdmax) pa->glsuspectdmax = testll; + } + + trie_insert(pa->dirs, incsuspect, strlen(incsuspect)); + cntd++; + break; + case 'f': case 'l': + if (cntfl==0) { + pa->glsuspectflmin = testll; + pa->glsuspectflmax = testll; + } else { + if (testll < pa->glsuspectflmin) pa->glsuspectflmin = testll; + if (testll > pa->glsuspectflmax) pa->glsuspectflmax = testll; + } + + trie_insert(pa->fls, incsuspect, strlen(incsuspect)); + cntfl++; + break; } - //printf("insuspect %s %s %lld %lld\n",incsuspect, incsuspecttype,glsuspectdmin,glsuspectdmax ); - trie_insert(headd, incsuspect, 1); - cntd++; - } - } - fclose(isf); + } + + fclose(isf); } - if (in->output == OUTDB) { - i=0; - while (i < in->maxthreads) { - SNPRINTF(outdbn,MAXPATH,"%s.%zu",in->outname.data,i); - gts.outdbd[i]=opendb(outdbn, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 1, 1 - , create_readdirplus_tables, NULL - #if defined(DEBUG) && defined(PER_THREAD_STATS) - , NULL, NULL - , NULL, NULL - #endif - ); - global_res[i]=insertdbprep(gts.outdbd[i], READDIRPLUS_INSERT); - if (in->stride > 0) { - if (pthread_mutex_init(&outdb_mutex[i], NULL) != 0) { - fprintf(stderr,"\n mutex %zu init failed\n", i); - } + pa->ta = calloc(pa->in.maxthreads, sizeof(struct ThreadArgs)); + + if (pa->in.output == OUTDB) { + for(size_t i = 0; i < pa->in.maxthreads; i++) { + struct ThreadArgs *ta = &pa->ta[i]; + SNPRINTF(outname, MAXPATH, "%s.%zu", pa->in.outname.data, i); + ta->db = opendb(outname, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 1, 1 + , create_readdirplus_tables, NULL + #if defined(DEBUG) && defined(PER_THREAD_STATS) + , NULL, NULL + , NULL, NULL + #endif + ); + ta->res = insertdbprep(ta->db, READDIRPLUS_INSERT); } - i++; - } } - - //open up the output files if needed - if (in->output == OUTFILE) { - i=0; - while (i < in->maxthreads) { - SNPRINTF(outfn,MAXPATH,"%s.%zu",in->outname.data,i); - //fprintf(stderr,"init opening %s.%d",in.outname,i); - gts.outfd[i]=fopen(outfn,"w"); - if (in->stride > 0) { - if (pthread_mutex_init(&outfile_mutex[i], NULL) != 0) { - fprintf(stderr,"\n mutex %zu init failed\n", i); - } + else if (pa->in.output == OUTFILE) { + for(size_t i = 0; i < pa->in.maxthreads; i++) { + struct ThreadArgs *ta = &pa->ta[i]; + SNPRINTF(outname, MAXPATH, "%s.%zu", pa->in.outname.data, i); + ta->file = fopen(outname, "w"); } - i++; - } } - // process input directory and put it on the queue - SNPRINTF(mywork->name,MAXPATH,"%s",in->name.data); - if (access(in->name.data, R_OK | X_OK)) { - fprintf(stderr, "couldn't access input dir '%s': %s\n", - in->name.data, strerror(errno)); - return 1; - } - struct stat st; - lstat(in->name.data, &st); - if (!S_ISDIR(st.st_mode) ) { - fprintf(stderr,"input-dir '%s' is not a directory\n", in->name.data); - return 1; + if (pa->in.stride > 0) { + for(size_t i = 0; i < pa->in.maxthreads; i++) { + struct ThreadArgs *ta = &pa->ta[i]; + pthread_mutex_init(&ta->mutex, NULL); + } } - // set top parent inode to zero + struct work *mywork = calloc(1, sizeof(struct work)); + + mywork->name_len = SNFORMAT_S(mywork->name, sizeof(mywork->name), 1, pa->in.name.data, pa->in.name.len); + QPTPool_enqueue(ctx, 0, processdir, mywork); return 0; } -int processfin(struct input *in) { - size_t i; - - // close output dbs here - if (in->output == OUTDB) { - i=0; - while (i < in->maxthreads) { - insertdbfin(global_res[i]); - closedb(gts.outdbd[i]); - if (in->stride > 0) { - pthread_mutex_destroy(&outdb_mutex[i]); - } - i++; - } - } +int processfin(struct PoolArgs *pa) { + for(size_t i = 0; i < pa->in.maxthreads; i++) { + struct ThreadArgs *ta = &pa->ta[i]; + if (ta->file) { + fclose(ta->file); + } - // close outputfiles - if (in->output == OUTFILE) { - i=0; - while (i < in->maxthreads) { - fclose(gts.outfd[i]); - if (in->stride > 0) { - pthread_mutex_destroy(&outdb_mutex[i]); - } - i++; - } - } + insertdbfin(ta->res); + closedb(ta->db); - trie_free(headfl); - trie_free(headd); + if (pa->in.stride > 0) { + pthread_mutex_destroy(&ta->mutex); + } + } - return 0; + trie_free(pa->fls); + trie_free(pa->dirs); + free(pa->ta); + + return 0; } -// This app allows users to do a readdirplus walk and optionally print dirs, print links/files, create outputdb -int validate_inputs(struct input *in) { - if (in->buildindex && in->nameto.data[0]) { - fprintf(stderr, "In bfwreaddirplus2db building an index '-b' the index must go into the src tree\n"); - fprintf(stderr, "and -t means you are specifying a parking lot directory for gufi directory db's to be put under their znumber\n"); - fprintf(stderr, "for incremental operations by inode\n"); - return -1; - } - - if (in->buildindex) { - fprintf(stderr,"You are putting the index dbs in input directory\n"); - in->buildinindir = 1; - in->nameto = in->name; - } - return 0; +static int validate_inputs(struct input *in) { + if (in->buildindex) { + fprintf(stderr, "You are putting the index dbs in input directory\n"); + in->nameto = in->name; + } + return 0; } -void sub_help() { - printf("input_dir walk this tree to produce GUFI index\n"); - printf("\n"); +static void sub_help() { + printf("input_dir walk this tree to produce GUFI index\n"); + printf("index build GUFI index here\n"); + printf("\n"); } -int main(int argc, char *argv[]) -{ - struct stat st; - int rc; - - // process input args - all programs share the common 'struct input', - // but allow different fields to be filled at the command-line. - // Callers provide the options-string for get_opt(), which will - // control which options are parsed for each program. - //fprintf(stderr,"in main beforeparse\n"); - struct input in; - int idx = parse_cmd_line(argc, argv, "hHn:O:ro:d:RYZW:g:A:c:xbt:", 1, "input_dir", &in); - //fprintf(stderr,"in main right after parse\n"); - if (in.helped) +int main(int argc, char *argv[]) { + /* process input args - all programs share the common 'struct input', */ + /* but allow different fields to be filled at the command-line. */ + /* Callers provide the options-string for get_opt(), which will */ + /* control which options are parsed for each program. */ + struct PoolArgs pa; + memset(&pa, 0, sizeof(pa)); + int idx = parse_cmd_line(argc, argv, "hHn:O:o:d:rRYZW:g:A:c:xb", 1, "input_dir [index]", &pa.in); + if (pa.in.helped) sub_help(); - if (idx < 0) - return -1; - else { - // parse positional args, following the options - INSTALL_STR(&in.name, argv[idx++]); - } - //fprintf(stderr,"in main after parse\n"); - if (validate_inputs(&in)) + if (idx < 0) + return -!pa.in.helped; + else { + INSTALL_STR(&pa.in.name, argv[idx++]); + + if (idx < argc) { + INSTALL_STR(&pa.in.nameto, argv[idx++]); + } + } + + if (validate_inputs(&pa.in)) { return -1; + } - //fprintf(stderr,"in main after validate\n"); - /* check the output directory for the gufi dbs for suspect dirs if provided */ - gltodirmode=0; - rc=1; - if (strlen(in.nameto.data) > 0) { - gltodirmode=1; - /*make sure the directory to put the gufi dbs into exists and we can write to it */ - rc=lstat(in.nameto.data,&st); - if (rc != 0) { - fprintf(stdout,"directory to place gufi dbs problem for %s\n",in.nameto.data); - return -1; - } - if (!S_ISDIR(st.st_mode) ) { - fprintf(stdout,"directory to place gufi dbs is not a directory\n"); - return -1; - } - } + /* check the output directory for the gufi dbs for suspect dirs if provided */ + pa.gltodirmode = 0; + if (pa.in.nameto.len > 0) { + pa.gltodirmode = 1; + /* make sure the directory to put the gufi dbs into exists and we can write to it */ + struct stat st; + if (lstat(pa.in.nameto.data, &st) != 0) { + fprintf(stdout, "directory to place gufi dbs problem for %s\n", pa.in.nameto.data); + return -1; + } + if (!S_ISDIR(st.st_mode)) { + fprintf(stdout, "directory to place gufi dbs is not a directory\n"); + return -1; + } + } - if (in.buildinindir == 1) gltodirmode=1; + if (pa.in.buildindex) { + pa.gltodirmode = 1; + } - QPTPool_t *pool = QPTPool_init(in.maxthreads, NULL); + QPTPool_t *pool = QPTPool_init(pa.in.maxthreads, &pa); if (QPTPool_start(pool) != 0) { fprintf(stderr, "Error: Failed to start thread pool\n"); QPTPool_destroy(pool); return -1; } - processinit(&in, pool); + processinit(&pa, pool); - QPTPool_wait(pool); + QPTPool_wait(pool); - QPTPool_destroy(pool); + QPTPool_destroy(pool); - processfin(&in); + processfin(&pa); - return 0; + return 0; } diff --git a/src/dbutils.c b/src/dbutils.c index 7a309cdf1..17edc1d22 100644 --- a/src/dbutils.c +++ b/src/dbutils.c @@ -603,39 +603,6 @@ int insertdbgo_xattrs(struct input *in, struct stat *dir, struct entry_data *ed, return 0; } -int insertdbgor(struct work *pwork, struct entry_data *ed, sqlite3_stmt *res) -{ - int rc = 0; - char *zname = sqlite3_mprintf("%q", pwork->name); - char *ztype = sqlite3_mprintf("%q", ed->type); - int error = sqlite3_bind_text(res, 1, zname, -1, SQLITE_TRANSIENT); - if (error != SQLITE_OK) { - fprintf(stderr, "SQL insertdbgor bind name: %s error %d err %s\n", - pwork->name, error, sqlite3_errstr(error)); - rc = 1; - goto cleanup; - } - sqlite3_bind_text(res, 2, ztype, -1, SQLITE_TRANSIENT); - sqlite3_bind_int64(res, 3, ed->statuso.st_ino); - sqlite3_bind_int64(res, 4, pwork->pinode); - sqlite3_bind_int64(res, 5, ed->suspect); - - error = sqlite3_step(res); - if (error != SQLITE_DONE) { - fprintf(stderr, "SQL insertdbgor step: %s error %d err %s\n", - pwork->name, error, sqlite3_errstr(error)); - rc = 1; - } - - cleanup: - sqlite3_free(zname); - sqlite3_free(ztype); - sqlite3_reset(res); - sqlite3_clear_bindings(res); - - return rc; -} - int insertsumdb(sqlite3 *sdb, const char *path, struct work *pwork, struct entry_data *ed, struct sum *su) { sqlite3_stmt *res = insertdbprep(sdb, SUMMARY_INSERT); diff --git a/src/external.c b/src/external.c index e1f09bd13..08824deb9 100644 --- a/src/external.c +++ b/src/external.c @@ -76,6 +76,7 @@ const char EXTERNAL_DBS_PWD_INSERT[] = "INSERT INTO " EXTERNAL_DBS_PWD " VALUES (@filename, @attachname, @mode, @uid, @gid);"; const char EXTERNAL_DBS_ROLLUP_CREATE[] = + "DROP TABLE IF EXISTS " EXTERNAL_DBS_ROLLUP ";" "CREATE TABLE " EXTERNAL_DBS_ROLLUP "(filename TEXT, attachname TEXT, mode INT64, uid INT64, gid INT64, PRIMARY KEY(filename, attachname));"; const char EXTERNAL_DBS_ROLLUP_INSERT[] = diff --git a/src/template_db.c b/src/template_db.c index 965d928f7..833ed048e 100644 --- a/src/template_db.c +++ b/src/template_db.c @@ -160,7 +160,7 @@ static int create_xattr_tables(const char *name, sqlite3 *db, void *args) { return 0; } -static int create_dbdb_tables(const char *name, sqlite3 *db, void *args) { +int create_dbdb_tables(const char *name, sqlite3 *db, void *args) { if ((create_table_wrapper(name, db, ENTRIES, ENTRIES_CREATE) != SQLITE_OK) || (create_table_wrapper(name, db, SUMMARY, SUMMARY_CREATE) != SQLITE_OK) || (create_table_wrapper(name, db, VRSUMMARY, VRSUMMARY_CREATE) != SQLITE_OK) || diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 931d8e266..a8106f150 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -95,21 +95,16 @@ foreach(TEST runuidgidummary runuserfilespacehog runuserfilespacehogusesummary - runtests - testdir.tar) + runtests) # copy the script into the build directory for easy access configure_file("${TEST}" "${TEST}" @ONLY) endforeach() -# don't modify the tarball -configure_file("testdir.tar" "testdir.tar" COPYONLY) - # allow for the test working directory to be moved set(TEST_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" CACHE PATH "Directory to run tests in") execute_process(COMMAND "${CMAKE_COMMAND}" -E make_directory "${TEST_WORKING_DIRECTORY}") -# create an empty directory and extract testdir.tar into it for runtests -set(TESTTAR "${CMAKE_CURRENT_BINARY_DIR}/testdir.tar") +set(TESTTAR "${CMAKE_CURRENT_SOURCE_DIR}/testdir.tar") set(TESTDIR "${TEST_WORKING_DIRECTORY}/testdir") set(TESTDST "${TEST_WORKING_DIRECTORY}/gary") diff --git a/test/runbfwreaddirplus2db b/test/runbfwreaddirplus2db index 10ee40c98..e77f86341 100755 --- a/test/runbfwreaddirplus2db +++ b/test/runbfwreaddirplus2db @@ -62,10 +62,12 @@ -SCRIPT="$BASH_SOURCE" # Must be at the top -DIR=`dirname $SCRIPT` -BFWREADDIRPLUS2DB=$DIR/../src/bfwreaddirplus2db -QUERYDBS=$DIR/../src/querydbs +set -e + +SCRIPT="${BASH_SOURCE[0]}" # Must be at the top +DIR="$(dirname ${SCRIPT})" +BFWREADDIRPLUS2DB="$(realpath ${DIR}/../src/bfwreaddirplus2db)" +QUERYDBS="$(realpath ${DIR}/../src/querydbs)" # NOTE: The destination GUFI-tree is optional @@ -100,12 +102,12 @@ usage() { exit -1 } -SOURCE=$1 +SOURCE="$(realpath $1)" -if [[ $# -ne 1 ]]; then - usage $0 -elif [ ! -d $SOURCE ]; then - usage $0 +if [[ "$#" -lt 1 ]]; then + usage "$0" +elif [[ ! -d "${SOURCE}" ]]; then + usage "$0" fi @@ -120,23 +122,23 @@ fi # ........................................................................... -# defaults -DEST=$1 - # run bfwreaddirplus2db and send output to screen echo echo "out to screen" -echo $BFWREADDIRPLUS2DB -n 1 $DEST -$BFWREADDIRPLUS2DB -n 1 $DEST +echo "${BFWREADDIRPLUS2DB}" -h +"${BFWREADDIRPLUS2DB}" -h + +echo "${BFWREADDIRPLUS2DB}" -n 1 "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 1 "${SOURCE}" # run to create output db echo echo "send to output to a db both files and dirs" rm -f bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 1 -r -R -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 1 -r -R -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 1 -r -R -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 1 -r -R -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' @@ -146,8 +148,8 @@ echo echo "send to output to a db dirs only" rm bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 1 -R -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 1 -R -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 1 -R -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 1 -R -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' @@ -157,8 +159,8 @@ echo echo "send to output to a db files and links only" rm -f bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 1 -r -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 1 -r -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 1 -r -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 1 -r -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' @@ -168,41 +170,41 @@ echo echo "send to output to a db both files and dirs with all dirs suspect 2 threads" rm bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 2 -r -R -Y -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 2 -r -R -Y -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -r -R -Y -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 2 -r -R -Y -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* -echo "" +echo echo "thread 0 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' -echo "" +echo echo "thread 1 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.1 'select * from readdirplus;' -echo "" +echo # run querydbs to union all the db's together into one table echo " run querydbs to union all the db's together into one table" -echo "" -$QUERYDBS -N -V bfwreaddirplusdb 2 "select * from vreaddirplus" readdirplus +echo +$QUERYDBS -N -V readdirplus "select * from vreaddirplus" bfwreaddirplusdb.* # run to create output db echo echo "send to output to a db both files and dirs with all dirs suspect 2 threads stride 10" rm bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 2 -r -R -Y -g 10 -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 2 -r -R -Y -g 10 -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -r -R -Y -g 10 -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 2 -r -R -Y -g 10 -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* -echo "" +echo echo "thread 0 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' -echo "" +echo echo "thread 1 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.1 'select * from readdirplus;' @@ -210,24 +212,23 @@ sqlite3 -line bfwreaddirplusdb.1 'select * from readdirplus;' echo echo "send to output db dirs only 2 threads suspectmethod 1 insuspectdfile" -rm insuspectfile -#stat testdir/a | @AWK@ ' { print $2 " f" } ' >> insuspectfile -ls -i testdir/a | @AWK@ ' { print $1 " f" } ' >> insuspectfile -#stat testdir/c | @AWK@ ' { print $2 " d" } ' >> insuspectfile -ls -id testdir/c | @AWK@ ' { print $1 " d" } ' >> insuspectfile +( + stat -c '%i f' testdir/a + stat -c '%i d' testdir/c +) > insuspectfile rm bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 2 -R -A 1 -W insuspectfile -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 2 -r -R -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -R -A 1 -W insuspectfile -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 2 -r -R -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* -echo "" +echo echo "thread 0 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' -echo "" +echo echo "thread 1 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.1 'select * from readdirplus;' @@ -236,17 +237,17 @@ echo echo "send to output to a db only dirs 2 threads suspectmethod 1 stride 10 insuspectfile" rm bfwreaddirplusdb.* -echo $BFWREADDIRPLUS2DB -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb $DEST -$BFWREADDIRPLUS2DB -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb "${SOURCE}" ls -l bfwreaddirplusdb.* -echo "" +echo echo "thread 0 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' -echo "" +echo echo "thread 1 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.1 'select * from readdirplus;' @@ -257,64 +258,68 @@ echo "send to output to a db only dirs 2 threads suspectmethod 1 stride 10 insus rm bfwreaddirplusdb.* rm -rf testbwrpdb mkdir testbwrpdb -echo $BFWREADDIRPLUS2DB -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb -x -t testbwrpdb $DEST -$BFWREADDIRPLUS2DB -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb -x -t testbwrpdb $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb -x "${SOURCE}" testbwrpdb +"${BFWREADDIRPLUS2DB}" -n 2 -R -A 1 -g 10 -W insuspectfile -O bfwreaddirplusdb -x "${SOURCE}" testbwrpdb ls -l bfwreaddirplusdb.* -echo "" +echo echo "thread 0 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.0 'select * from readdirplus;' -echo "" +echo echo "thread 1 outdb" -echo "" +echo sqlite3 -line bfwreaddirplusdb.1 'select * from readdirplus;' ls -l testbwrpdb #numpl=`ls -l testbwrpdb | wc -l` #if [ $numpl -gt 0 ] #then -t1=`ls testbwrpdb | head -1 | @AWK@ ' { print $1 } '` -sqlite3 -line testbwrpdb/$t1 '.tables' -sqlite3 -line testbwrpdb/$t1 'select * from entries;' -sqlite3 -line testbwrpdb/$t1 'select * from summary;' -t2=`ls testbwrpdb | tail -1 | @AWK@ ' { print $1 } '` -sqlite3 -line testbwrpdb/$t1 '.tables' -sqlite3 -line testbwrpdb/$t2 'select * from entries;' -sqlite3 -line testbwrpdb/$t2 'select * from summary;' +t1="$(ls -1 testbwrpdb | head -1)" +sqlite3 -line "testbwrpdb/${t1}" '.tables' +sqlite3 -line "testbwrpdb/${t1}" 'select * from entries;' +sqlite3 -line "testbwrpdb/${t1}" 'select * from summary;' +t2="$(ls -1 testbwrpdb | tail -1)" +sqlite3 -line "testbwrpdb/${t2}" '.tables' +sqlite3 -line "testbwrpdb/${t2}" 'select * from entries;' +sqlite3 -line "testbwrpdb/${t2}" 'select * from summary;' #fi +rm -r testbwrpdb +rm bfwreaddirplusdb.* # run to create output files echo echo "send to output to files files and dirs 2 threads suspectmethod 0 " -rm bfwreaddirplusfl.* -echo $BFWREADDIRPLUS2DB -n 2 -A 0 -o bfwreaddirplusfl $DEST -$BFWREADDIRPLUS2DB -n 2 -o bfwreaddirplusfl $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -A 0 -o bfwreaddirplusfl "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 2 -o bfwreaddirplusfl "${SOURCE}" ls -l bfwreaddirplusfl.* -echo "" +echo echo "thread 0 outfl" -echo "" +echo cat bfwreaddirplusfl.0 -echo "" +echo echo "thread 1 outfl" cat bfwreaddirplusfl.1 -echo "" +echo # run to create output files with stride echo echo "send to output to files files and dirs 2 threads suspectmethod 0 stride 10 " rm bfwreaddirplusfl.* -echo $BFWREADDIRPLUS2DB -n 2 -A 0 -g 10 -o bfwreaddirplusfl $DEST -$BFWREADDIRPLUS2DB -n 2 -o bfwreaddirplusfl $DEST +echo "${BFWREADDIRPLUS2DB}" -n 2 -A 0 -g 10 -o bfwreaddirplusfl "${SOURCE}" +"${BFWREADDIRPLUS2DB}" -n 2 -o bfwreaddirplusfl "${SOURCE}" ls -l bfwreaddirplusfl.* -echo "" +echo echo "thread 0 outfl" -echo "" +echo cat bfwreaddirplusfl.0 -echo "" +echo echo "thread 1 outfl" cat bfwreaddirplusfl.1 -echo "" +echo + +rm bfwreaddirplusfl.* +rm insuspectfile diff --git a/test/runtests b/test/runtests index 76c42b647..ad62cd131 100755 --- a/test/runtests +++ b/test/runtests @@ -62,8 +62,10 @@ +set -e + TESTS="$(dirname ${BASH_SOURCE[0]})" # Must be at the top -set -x + usage() { echo "Usage: $0 tar dir GUFI_tree" } @@ -95,7 +97,7 @@ ${TESTS}/runbfwi ${TREE} ${TARGET} ${TESTS}/runbfti ${INDEX} ${TESTS}/runbfq ${INDEX} ${TESTS}/runquerydbs ${INDEX} 2 -${TESTS}/runbfwreaddirplus2db ${INDEX} +${TESTS}/runbfwreaddirplus2db ${TREE} ${TESTS}/rundfw ${INDEX} # other tests