Skip to content

builder: defer submodule detection to git #507

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 11, 2024
Merged

builder: defer submodule detection to git #507

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b356581
builder: defer submodule detection to git
abn Nov 6, 2024
81f3765
chore(lint): fix pylint errors
abn Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 31 additions & 55 deletions src/tito/builder/submodule_aware_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
find_spec_like_file,
get_commit_timestamp,
)
from tito.exception import TitoException
from tito.tar import TarFixer


Expand Down Expand Up @@ -109,54 +108,31 @@ def run_git_archive(self, relative_git_dir, prefix, commit, dest_tar, subdir=Non
"%s > /dev/null" % git_archive_cmd,
)

# pylint: disable=too-many-locals, too-many-arguments, consider-using-f-string
def _submodule_archives(self, relative_git_dir, prefix, commit, initial_tar,
submodule_tree="."):
with chdir(submodule_tree):
submodules_cmd = "git config --file .gitmodules --get-regexp path"
submodules_output = run_command(submodules_cmd)

# split submodules output on newline
# then on tab, and the directory is the last entry
submodules_list = [
line.split(" ")[-1] for line in submodules_output.split("\n")
]

# We ignore the hash in the sub modules list as we'll have to get the correct one
# from the commit id in commit
for submodule in submodules_list:
with chdir(submodule_tree):
submodule_git_dir = os.path.join(submodule, ".git")
if not os.path.exists(submodule_git_dir):
raise TitoException("%r path does not contain '.git'. "
"Have you cloned the repository recursively?\n"
"Try `git submodule update --init --recursive`!" %
os.path.abspath(submodule))
# to find the submodule shars:
# git rev-parse <commit>:./<submodule>
rev_parse_cmd = "git rev-parse %s:./%s" % (commit, submodule)
submodule_commit = run_command(rev_parse_cmd)
submodule_tar_file = "%s.%s" % (initial_tar, submodule.replace("/", "_"))
# prefix should be <prefix>/<submodule>
submodule_prefix = "%s/%s" % (prefix, submodule)

self.run_git_archive(
relative_git_dir,
submodule_prefix,
submodule_commit,
submodule_tar_file,
submodule,
)
yield (submodule_tar_file)

submodule_dir = os.path.join(submodule_tree, submodule)
gitmodules_path = os.path.join(submodule_dir, ".gitmodules")
if os.path.exists(gitmodules_path):
for archive in self._submodule_archives(
relative_git_dir, submodule_prefix, submodule_commit,
submodule_tar_file, submodule_dir
):
yield archive
def _submodule_archives(self, relative_git_dir, prefix, initial_tar, source_tree="."):
with chdir(source_tree):
# Let git handle edge cases for .gitmodules (eg: empty files etc)
submodules_status_cmd = "git submodule status --recursive"
submodules_status_output = run_command(submodules_status_cmd)

for line in submodules_status_output.strip().split("\n"):
row = line.split()
submodule_commit, submodule_relative_dir = row[0], row[1]

submodule_tar_file_suffix = submodule_relative_dir.replace("/", "_")
submodule_tar_file = "{0}.{1}".format(initial_tar, submodule_tar_file_suffix)

# prefix should be <prefix>/<submodule>
submodule_prefix = "{0}/{1}".format(prefix, submodule_relative_dir)

self.run_git_archive(
relative_git_dir,
submodule_prefix,
submodule_commit,
submodule_tar_file,
submodule_relative_dir,
)

yield submodule_tar_file

def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz):
"""
Expand All @@ -176,12 +152,12 @@ def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz):
timestamp = get_commit_timestamp(commit)

# Accommodate standalone projects with specfile in root of git repo:
relative_git_dir = "%s" % relative_dir
relative_git_dir = str(relative_dir)
if relative_git_dir in ["/", "./"]:
relative_git_dir = ""

basename = os.path.splitext(dest_tgz)[0]
initial_tar = "%s.initial" % basename
initial_tar = "{0}.initial".format(basename)

# We need to tar up the following:
# 1. the current repo
Expand All @@ -190,11 +166,11 @@ def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz):
# 2. all of the submodules
# then combine those into a single archive.
for submodule_tar_file in self._submodule_archives(
relative_git_dir, prefix, commit, initial_tar
relative_git_dir, prefix, initial_tar
):
run_command("tar -Af %s %s" % (initial_tar, submodule_tar_file))
run_command("tar -Af {0} {1}".format(initial_tar, submodule_tar_file))

fixed_tar = "%s.tar" % basename
fixed_tar = "{0}.tar".format(basename)
fixed_tar_fh = open(fixed_tar, "wb")
try:
tarfixer = TarFixer(
Expand All @@ -205,4 +181,4 @@ def create_tgz(self, git_root, prefix, commit, relative_dir, dest_tgz):
fixed_tar_fh.close()

# It's a pity we can't use Python's gzip, but it doesn't offer an equivalent of -n
return run_command("gzip -n -c < %s > %s" % (fixed_tar, dest_tgz))
return run_command("gzip -n -c < {0} > {1}".format(fixed_tar, dest_tgz))
Loading