Support GIT upstream
Add the support to check how far a project is behind upstream git.
Still need to manually run git merge. But this tool can help update
METADATA. :)
Test: ./updater.sh update perf_data_converter
Change-Id: Ic46a0eb723ae22f0fc7d61a67a14299761b564a4
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..bee8a64
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+__pycache__
diff --git a/external_updater.py b/external_updater.py
index 4200676..4ceaa21 100644
--- a/external_updater.py
+++ b/external_updater.py
@@ -20,15 +20,17 @@
import argparse
import os
+import subprocess
from google.protobuf import text_format # pylint: disable=import-error
import fileutils
+from git_updater import GitUpdater
from github_archive_updater import GithubArchiveUpdater
import updater_utils
-UPDATERS = [GithubArchiveUpdater]
+UPDATERS = [GithubArchiveUpdater, GitUpdater]
def color_string(string, color):
@@ -82,21 +84,28 @@
end='')
updater = build_updater(proj_path)
if updater is None:
- return
+ return (None, None)
try:
- latest = updater.get_latest_version()
- current = updater.get_current_version()
- except IOError as e:
- print('{} {}.'.format(color_string('Failed to check latest version.',
- 'FAILED'),
- e))
- return
-
- if current != latest:
- print('{} Current version: {}. Latest version: {}.'. format(
- color_string('New version found.', 'SUCCESS'), current, latest))
- else:
- print('No new version. Current version: {}.'.format(latest))
+ new_version = updater.check()
+ if new_version:
+ print(color_string(' New version found.', 'SUCCESS'))
+ else:
+ print(' No new version.')
+ return (updater, new_version)
+ except IOError as err:
+ print('{} {}.'.format(color_string('Failed.', 'FAILED'),
+ err))
+ return (None, None)
+ except subprocess.CalledProcessError as err:
+ print(
+ '{} {}\nstdout: {}\nstderr: {}.'.format(
+ color_string(
+ 'Failed.',
+ 'FAILED'),
+ err,
+ err.stdout,
+ err.stderr))
+ return (None, None)
def check(args):
@@ -108,33 +117,12 @@
def update(args):
"""Handler for update command."""
- updater = build_updater(args.path)
+ updater, new_version = check_update(args.path)
if updater is None:
return
- try:
- latest = updater.get_latest_version()
- current = updater.get_current_version()
- except IOError as e:
- print('{} {}.'.format(
- color_string('Failed to check latest version.',
- 'FAILED'),
- e))
+ if not new_version and not args.force:
return
- if current == latest and not args.force:
- print(
- '{} for {}. Current version {} is latest. '
- 'Use --force to update anyway.'.format(
- color_string(
- 'Nothing to update',
- 'FAILED'),
- args.path,
- current))
- return
-
- print('{} from version {} to version {}.{}'.format(
- color_string('Updating', 'SUCCESS'), args.path, current, latest))
-
updater.update()
diff --git a/git_updater.py b/git_updater.py
new file mode 100644
index 0000000..bde48a7
--- /dev/null
+++ b/git_updater.py
@@ -0,0 +1,118 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to check updates from Git upstream."""
+
+
+import datetime
+
+import fileutils
+import git_utils
+import metadata_pb2 # pylint: disable=import-error
+
+
+class GitUpdater():
+ """Updater for Git upstream."""
+
+ def __init__(self, url, proj_path, metadata):
+ if url.type != metadata_pb2.URL.GIT:
+ raise ValueError('Only support GIT upstream.')
+ self.proj_path = proj_path
+ self.metadata = metadata
+ self.upstream_url = url
+ self.upstream_remote_name = None
+ self.android_remote_name = None
+ self.latest_commit = None
+
+ def _setup_remote(self):
+ remotes = git_utils.list_remotes(self.proj_path)
+ for name, url in remotes.items():
+ if url == self.upstream_url.value:
+ self.upstream_remote_name = name
+
+ # Guess android remote name.
+ if '/platform/external/' in url:
+ self.android_remote_name = name
+
+ if self.upstream_remote_name is None:
+ self.upstream_remote_name = "update_origin"
+ git_utils.add_remote(self.proj_path, self.upstream_remote_name,
+ self.upstream_url.value)
+
+ git_utils.fetch(self.proj_path,
+ [self.upstream_remote_name, self.android_remote_name])
+
+ def check(self):
+ """Checks upstream and returns whether a new version is available."""
+
+ self._setup_remote()
+ commits = git_utils.get_commits_ahead(
+ self.proj_path, self.upstream_remote_name + '/master',
+ self.android_remote_name + '/master')
+
+ if not commits:
+ return False
+
+ self.latest_commit = commits[0]
+ commit_time = git_utils.get_commit_time(self.proj_path, commits[-1])
+ time_behind = datetime.datetime.now() - commit_time
+ print('{} commits ({} days) behind.'.format(
+ len(commits), time_behind.days), end='')
+ return True
+
+ def _write_metadata(self, path):
+ updated_metadata = metadata_pb2.MetaData()
+ updated_metadata.CopyFrom(self.metadata)
+ updated_metadata.third_party.version = self.latest_commit
+ fileutils.write_metadata(path, updated_metadata)
+
+ def update(self):
+ """Updates the package.
+
+ Has to call check() before this function.
+ """
+ # See whether we have a local upstream.
+ branches = git_utils.list_remote_branches(
+ self.proj_path, self.android_remote_name)
+ upstreams = [
+ branch for branch in branches if branch.startswith('upstream-')]
+ if len(upstreams) == 1:
+ merge_branch = '{}/{}'.format(
+ self.android_remote_name, upstreams[0])
+ elif not upstreams:
+ merge_branch = 'update_origin/master'
+ else:
+ raise ValueError('Ambiguous upstream branch. ' + upstreams)
+
+ upstream_branch = self.upstream_remote_name + '/master'
+
+ commits = git_utils.get_commits_ahead(
+ self.proj_path, merge_branch, upstream_branch)
+ if commits:
+ print('Warning! {} is {} commits ahead of {}. {}'.format(
+ merge_branch, len(commits), upstream_branch, commits))
+
+ commits = git_utils.get_commits_ahead(
+ self.proj_path, upstream_branch, merge_branch)
+ if commits:
+ print('Warning! {} is {} commits behind of {}.'.format(
+ merge_branch, len(commits), upstream_branch))
+
+ self._write_metadata(self.proj_path)
+ print("""
+This tool only updates METADATA. Run the following command to update:
+ git merge {merge_branch}
+
+To check all local changes:
+ git diff {merge_branch} HEAD
+""".format(merge_branch=merge_branch))
diff --git a/git_utils.py b/git_utils.py
new file mode 100644
index 0000000..8e8f96d
--- /dev/null
+++ b/git_utils.py
@@ -0,0 +1,77 @@
+# Copyright (C) 2018 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the 'License');
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''Helper functions to communicate with Git.'''
+
+import datetime
+import subprocess
+
+
+def _run(cmd, cwd):
+ """Runs a command with stdout and stderr redirected."""
+ return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ check=True, cwd=cwd)
+
+def fetch(proj_path, remote_names):
+ """Runs git fetch.
+
+ Args:
+ proj_path: Path to Git repository.
+ remote_names: Array of string to specify remote names.
+ """
+ _run(['git', 'fetch', '--multiple'] + remote_names, cwd=proj_path)
+
+def add_remote(proj_path, name, url):
+ """Adds a git remote.
+
+ Args:
+ proj_path: Path to Git repository.
+ name: Name of the new remote.
+ url: Url of the new remote.
+ """
+ _run(['git', 'remote', 'add', name, url], cwd=proj_path)
+
+def list_remotes(proj_path):
+ """Lists all Git remotes.
+
+ Args:
+ proj_path: Path to Git repository.
+
+ Returns:
+ A dict from remote name to remote url.
+ """
+ out = _run(['git', 'remote', '-v'], proj_path)
+ lines = out.stdout.decode('utf-8').splitlines()
+ return dict([line.split()[0:2] for line in lines])
+
+def get_commits_ahead(proj_path, branch, base_branch):
+ """Lists commits in `branch` but not `base_branch`."""
+ out = _run(['git', 'rev-list', '--left-only',
+ '{}...{}'.format(branch, base_branch)],
+ proj_path)
+ return out.stdout.decode('utf-8').splitlines()
+
+def get_commit_time(proj_path, commit):
+ """Gets commit time of one commit."""
+ out = _run(['git', 'show', '-s', '--format=%ct', commit], cwd=proj_path)
+ return datetime.datetime.fromtimestamp(int(out.stdout))
+
+def list_remote_branches(proj_path, remote_name):
+ """Lists all branches for a remote."""
+ out = _run(['git', 'branch', '-r'], cwd=proj_path)
+ lines = out.stdout.decode('utf-8').splitlines()
+ stripped = [line.strip() for line in lines]
+ remote_path = remote_name + '/'
+ remote_path_len = len(remote_path)
+ return [line[remote_path_len:] for line in stripped
+ if line.startswith(remote_path)]
diff --git a/github_archive_updater.py b/github_archive_updater.py
index 7f0bb77..3abe5be 100644
--- a/github_archive_updater.py
+++ b/github_archive_updater.py
@@ -58,7 +58,7 @@
except IndexError:
raise ValueError('Url format is not supported.')
- def get_latest_version(self):
+ def _get_latest_version(self):
"""Checks upstream and returns the latest version name we found."""
url = 'https://api.github.com/repos/{}/{}/releases/latest'.format(
@@ -67,7 +67,7 @@
self.data = json.loads(request.read().decode())
return self.data[self.VERSION_FIELD]
- def get_current_version(self):
+ def _get_current_version(self):
"""Returns the latest version name recorded in METADATA."""
return self.metadata.third_party.version
@@ -80,10 +80,21 @@
metadata_url.value = url
fileutils.write_metadata(path, updated_metadata)
+ def check(self):
+ """Checks update for package.
+
+ Returns True if a new version is available.
+ """
+ latest = self._get_latest_version()
+ current = self._get_current_version()
+ print('Current version: {}. Latest version: {}'.format(
+ current, latest), end='')
+ return current != latest
+
def update(self):
"""Updates the package.
- Has to call get_latest_version() before this function.
+ Has to call check() before this function.
"""
supported_assets = [
diff --git a/updater.sh b/updater.sh
index 2a3da75..4f03c89 100755
--- a/updater.sh
+++ b/updater.sh
@@ -16,5 +16,6 @@
cd $(dirname "$0")/../..
source build/envsetup.sh
+lunch aosp_arm-eng
mmma tools/external_updater
out/soong/host/linux-x86/bin/external_updater $@
