diff --git a/script/get-publish-history.py b/script/get-publish-history.py new file mode 100644 index 0000000..7c89bf1 --- /dev/null +++ b/script/get-publish-history.py @@ -0,0 +1,118 @@ +## Imports +import os +import sys +import git +import io +from pathlib import Path +import json +import pandas as pd + +## Module Constants +DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" +EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + + +def versions(path, branch='master'): + """ + This function returns a generator which iterates through all commits of + the repository located in the given path for the given branch. It yields + file diff information to show a timeseries of file changes. + """ + + # Create the repository, raises an error if it isn't one. + repo = git.Repo(path) + + # Iterate through every commit for the given branch in the repository + for commit in repo.iter_commits(branch): + # Determine the parent of the commit to diff against. + # If no parent, this is the first commit, so use empty tree. + # Then create a mapping of path to diff for each file changed. + parent = commit.parents[0] if commit.parents else EMPTY_TREE_SHA + diffs = { + diff.a_path: diff for diff in commit.diff(parent) + } + + # The stats on the commit is a summary of all the changes for this + # commit, we'll iterate through it to get the information we need. + for objpath, stats in commit.stats.files.items(): + + # Select the diff for the path in the stats + diff = diffs.get(objpath) + + # If the path is not in the dictionary, it's because it was + # renamed, so search through the b_paths for the current name. + if not diff: + for diff in diffs.values(): + if diff.b_path == path and diff.renamed: + break + + p = Path(objpath) + if len(p.parts) != 3: + print(f'skipping path: wrong depth ({p.parts})', file=sys.stderr) + continue + try: + obj = commit.tree / objpath + with io.BytesIO(obj.data_stream.read()) as f: + lastline = list(f.readlines())[-1].decode('utf-8') + except Exception as e: + print(f'failed to load file at commit {commit}', file=sys.stderr) + continue + + lastline = lastline.strip() + try: + d = json.loads(lastline) + except Exception as e: + print(f'failed to parse json at commit {commit}: {e}', file=sys.stderr) + continue + + row = { + 'path': os.path.join(path, objpath), + 'commit': commit.hexsha, + 'author': commit.author.email, + 'time': commit.authored_datetime.strftime(DATE_TIME_FORMAT), + #'size': diff_size(diff), + #'type': diff_type(diff), + 'crate': d['name'], + 'vers': d['vers'], + #'json': lastline, + } + + # Update the stats with the additional information + # stats.update(row) + # yield stats + + yield row + + +def diff_size(diff): + """ + Computes the size of the diff by comparing the size of the blobs. + """ + if diff.b_blob is None and diff.deleted_file: + # This is a deletion, so return negative the size of the original. + return diff.a_blob.size * -1 + + if diff.a_blob is None and diff.new_file: + # This is a new file, so return the size of the new value. + return diff.b_blob.size + + # Otherwise just return the size a-b + return diff.a_blob.size - diff.b_blob.size + + +def diff_type(diff): + """ + Determines the type of the diff by looking at the diff flags. + """ + if diff.renamed: return 'R' + if diff.deleted_file: return 'D' + if diff.new_file: return 'A' + return 'M' + +df = pd.DataFrame(versions('crate-index')) +df['time'] = pd.to_datetime(df['time']) +df['unix_nanos'] = df['time'].astype('int') +df = df.sort_values(by='time').groupby(['crate', 'vers']).last().reset_index() +buf = io.StringIO() +df.to_csv(buf, index=False) +print(buf.getvalue())