diff --git a/github-sync-tng/issue_command.rb b/github-sync-tng/issue_command.rb index 586675d..9084911 100755 --- a/github-sync-tng/issue_command.rb +++ b/github-sync-tng/issue_command.rb @@ -108,6 +108,8 @@ def get_labels(client, db, orgrepo) # [GitHub Client Calls = Count(Repo's Updated Issues + Repo's Updated Pull Requests (1 (for data) + 1 (for fix-merged-at) + pr-files) ] class SyncItemsCommand < BaseCommand + BLOCK_SIZE=50 + # params=(context, sync_db) def run(queue, *params) sync_items(queue, params[0], params[1], @args['org'], @args['repo']) @@ -116,7 +118,6 @@ def run(queue, *params) def sync_items(queue, context, issue_db, org, repo) orgrepo="#{org}/#{repo}" - issue_db.execute("BEGIN TRANSACTION"); maxTimestamp=db_getMaxTimestampForRepo(issue_db, org, repo) # Get the current max timestamp in the db if(maxTimestamp) # Increment the timestamp by a second to avoid getting repeats @@ -126,12 +127,24 @@ def sync_items(queue, context, issue_db, org, repo) issues=context.client.list_issues(orgrepo, { 'state' => 'all' } ) end unless(issues.empty?) - db_insert_issues(issue_db, issues, org, repo) # Insert any new items - db_link_issues(issue_db, issues, org, repo) - db_fix_merged_at(issue_db, context.client, issues, org, repo) # Put in PR specific data - namely merged_at - db_add_pull_request_files(issue_db, context.client, issues, org, repo) # Put in PR specific data - namely the files + their metrics + + while(issues.length>0) + + # Issue lists can be large when first importing, so handle in blocks + issue_block=issues.take(BLOCK_SIZE) + + issue_db.execute("BEGIN TRANSACTION"); + db_insert_issues(issue_db, issue_block, org, repo) # Insert any new items + db_link_issues(issue_db, issue_block, org, repo) + db_fix_merged_at(issue_db, context.client, issue_block, org, repo) # Put in PR specific data - namely merged_at + db_add_pull_request_files(issue_db, context.client, issue_block, org, repo) # Put in PR specific data - namely the files + their metrics + issue_db.execute("END TRANSACTION"); + + # remove the block just loaded + issues=issues.drop(BLOCK_SIZE) + end + end - issue_db.execute("END TRANSACTION"); end end