-
Notifications
You must be signed in to change notification settings - Fork 1
/
git_init.py
217 lines (167 loc) · 6.89 KB
/
git_init.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
import os, shutil, subprocess, sys, json
import requests
from github import Github
models_path = 'models/player/'
all_dirs = [dir for dir in os.listdir(models_path) if os.path.isdir(os.path.join(models_path,dir))]
all_dirs.sort()
total_dirs = len(all_dirs)
git_asset_root = '.git_data'
username = 'wootdata'
commit_user = 'wootguy'
commit_email = 'w00tguy123@gmail.com'
ssh_host_name = 'wootdata.github.com' # used to select an ssh key from ~/.ssh/config (this is an alias not a real host name)
num_buckets = 32
access_token = ''
with open('/home/pi/git_access_token.txt', 'r') as file:
access_token = file.read().replace('\n', '')
github = Github(access_token)
lowest_count = 99999
def hash_string(str):
hash = 0
for i in range(0, len(str)):
char = ord(str[i])
hash = ((hash<<5)-hash) + char
hash = hash % 15485863 # prevent hash ever increasing beyond 31 bits (prime)
return hash
def create_repos():
print("")
print("WARNING: This will delete all sc_models_* repos, locally and on GitHub.")
print("The repos will then be recreated which will take a long time.")
print("")
input("Press Enter to continue...")
if os.path.exists(git_asset_root):
shutil.rmtree(git_asset_root)
os.mkdir(git_asset_root)
print("Initializing asset repos")
for i in range(0, num_buckets):
git_path = os.path.join(git_asset_root, 'repo%s' % i)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'init']
subprocess.run(args)
# configure commit username/email
args = ['git', '--git-dir=%s' % git_path, 'config', 'user.name', '"%s"' % commit_user]
subprocess.run(args)
args = ['git', '--git-dir=%s' % git_path, 'config', 'user.email', '"%s"' % commit_email]
subprocess.run(args)
# Add files to each repo, balanced by hash key
print("Adding files to repos")
for idx, dir in enumerate(all_dirs):
b = hash_string(dir) % num_buckets
git_path = os.path.join(git_asset_root, 'repo%s' % b)
print("%s -> %s" % (dir, git_path))
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'add', os.path.join(models_path, dir), '-f']
subprocess.run(args)
# add common files and commit in all repos
for i in range(0, num_buckets):
git_path = os.path.join(git_asset_root, 'repo%s' % i)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'add', '.nojekyll']
subprocess.run(args)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'commit', '-m', 'initial commit']
subprocess.run(args)
github_user = github.get_user()
# Create repos, push to them, and enable github pages
for i in range(0, num_buckets):
git_path = os.path.join(git_asset_root, 'repo%s' % i)
repo_name = 'scmodels_data_%s' % i
try:
repo = github_user.get_repo(repo_name)
print("Deleting existing repo: %s" % repo_name)
if repo:
repo.delete()
except Exception as e:
print(e)
repo = github_user.create_repo(repo_name, description='storage partition for scmodels')
print("Created %s" % repo.full_name)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'remote', 'add', 'origin', 'git@%s:%s.git' % (ssh_host_name, repo.full_name)]
subprocess.run(args)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'push', '-u', 'origin', 'master']
subprocess.run(args)
print("Enabling GitHub Pages...")
headers = {
'Authorization': 'token ' + access_token,
'Accept': 'application/vnd.github.switcheroo-preview+json'
}
payload = {
'source': {
'branch': 'master',
'path': ''
}
}
resp = requests.post('https://api.github.com/repos/%s/%s/pages' % (username, repo_name), headers=headers, data=json.dumps(payload)).json()
print(resp)
print("Repo creation finished: %s" % repo_name)
print("")
def update(commit_message):
global all_dirs
if True:
# Add files to each repo, balanced by hash key
print("Adding files to repos")
if os.path.exists('updated.txt'):
with open("updated.txt", "r") as update_list:
all_dirs = update_list.readlines()
print("using updated.txt instead of checking all folders")
else:
print("Updating all folders because updated.txt does not exist (slow!)")
updated_buckets = []
for idx, dir in enumerate(all_dirs):
dir = dir.strip()
b = hash_string(dir) % num_buckets
updated_buckets.append(b)
git_path = os.path.join(git_asset_root, 'repo%s' % b)
print("%s -> %s" % (dir, git_path))
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'add', os.path.join(models_path, dir), '-f']
subprocess.run(args)
# commit and push
for i in range(0, num_buckets):
if i not in updated_buckets:
continue
repo_name = 'scmodels_data_%s' % i
print("\nUpdating %s" % repo_name)
git_path = os.path.join(git_asset_root, 'repo%s' % i)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'commit', '-m', commit_message]
subprocess.run(args)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'push']
subprocess.run(args)
if os.path.exists('updated.txt'):
os.remove("updated.txt")
def update_simple(commit_message):
global all_dirs
if True:
# Add files to each repo, balanced by hash key
print("Adding all changes to repos")
for b in range(num_buckets):
git_path = os.path.join(git_asset_root, 'repo%s' % b)
print("add all changes for %s" % (git_path))
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'add', '-u']
subprocess.run(args)
# commit and push
for i in range(0, num_buckets):
repo_name = 'scmodels_data_%s' % i
print("\nUpdating %s" % repo_name)
git_path = os.path.join(git_asset_root, 'repo%s' % i)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'commit', '-m', commit_message]
subprocess.run(args)
args = ['git', '--git-dir=%s' % git_path, '--work-tree=.', 'push']
subprocess.run(args)
args = sys.argv[1:]
if len(args) == 1 and args[0].lower() == 'help' or len(args) == 0:
print("\nUsage:")
print("python3 git_init.py [command]\n")
print("Available commands:")
print("create - creates or re-creates all data repos (takes like 8 hours)")
print("update 'commit message' - adds new models. Default commit message is 'add new models'")
print("update_rem 'commit message' - like update, but does 'git add -u' instead of adding new individual models.")
print(" This adds removed and updated files to the commit, ignoring untracked files.")
print("\nIMPORTANT: DO NOT RUN THIS AS ROOT")
if len(args) > 0:
if args[0].lower() == 'create':
create_repos()
if args[0].lower() == 'update':
commit_message = "add new models"
if len(args) > 1:
commit_message = args[1]
update(commit_message)
if args[0].lower() == 'update_rem':
commit_message = "add new models"
if len(args) > 1:
commit_message = args[1]
update_simple(commit_message)