bitbake: bitbake-setup: capture revisions while checking out layers

When initializing a build setup from a conf.json that only sets 'rev'
to a tag or branch, the actual revision would not be captured or
logged.

To capture the current layer state after an 'init' or 'update', the
checkout_layers function is extended to store the revision the
bb.fetch.Fetch pulled, and write that information into a
sources-fixed-revisions.json file. This file can then be fed back into
bitbake-setup init as: --sources-overrides

This new 'sources-fixed-revisions.json' is written during 'update_build' and
stored alongside the 'config-upstream.json' in the config dir. And put
with the later under version control by calling 'commit_config" after
'update_build'.

The use of 'deepcopy' is necessary to not modify the original input
data - which python passes around as reference.

(Bitbake rev: 95866ff03f78e987ae7e47daad053bc0f353eea4)

Signed-off-by: Johannes Schneider <johannes.schneider@leica-geosystems.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

bitbake/bin/bitbake-setup

@@ -18,6 +18,7 @@ import configparser
 import datetime
 import glob
 import subprocess
+import copy
 
 default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")
 
@@ -61,6 +62,12 @@ def write_upstream_config(config_dir, config_data):
     with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
         json.dump(config_data, s, sort_keys=True, indent=4)
 
+def write_sources_fixed_revisions(config_dir, config_data):
+    sources = {}
+    sources['sources'] = config_data
+    with open(os.path.join(config_dir, "sources-fixed-revisions.json"),'w') as s:
+        json.dump(sources, s, sort_keys=True, indent=4)
+
 def commit_config(config_dir):
     bb.process.run("git -C {} add .".format(config_dir))
     bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime()))
@@ -76,6 +83,7 @@ def _write_layer_list(dest, repodirs):
         json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)
 
 def checkout_layers(layers, layerdir, d):
+    layers_fixed_revisions = copy.deepcopy(layers)
     repodirs = []
     oesetupbuild = None
     print("Fetching layer/tool repositories into {}".format(layerdir))
@@ -99,6 +107,9 @@ def checkout_layers(layers, layerdir, d):
                 src_uri = f"{fetchuri};protocol={prot};rev={rev};nobranch=1;destsuffix={repodir}"
             fetcher = bb.fetch.Fetch([src_uri], d)
             do_fetch(fetcher, layerdir)
+            urldata = fetcher.ud[src_uri]
+            revision = urldata.revision
+            layers_fixed_revisions[r_name]['git-remote']['rev'] = revision
 
         if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
             oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
@@ -115,6 +126,8 @@ def checkout_layers(layers, layerdir, d):
                 os.remove(symlink)
             os.symlink(os.path.relpath(t,layerdir),symlink)
 
+    return layers_fixed_revisions
+
 def setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir):
     def _setup_build_conf(layers, build_conf_dir):
         os.makedirs(build_conf_dir)
@@ -269,15 +282,16 @@ def get_registry_config(registry_path, id):
     raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))
 
 def update_build(config, confdir, setupdir, layerdir, d):
-    layer_config = config["data"]["sources"]
+    layer_config = copy.deepcopy(config["data"]["sources"])
     layer_overrides = config["source-overrides"]["sources"]
     for k,v in layer_overrides.items():
         if k in layer_config:
             layer_config[k]["git-remote"] = v["git-remote"]
-    checkout_layers(layer_config, layerdir, d)
+    sources_fixed_revisions = checkout_layers(layer_config, layerdir, d)
     bitbake_config = config["bitbake-config"]
     thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
     setup_bitbake_build(bitbake_config, layerdir, setupdir, thisdir)
+    write_sources_fixed_revisions(confdir, sources_fixed_revisions)
 
 def int_input(allowed_values):
     n = None
@@ -471,8 +485,8 @@ def init_config(top_dir, settings, args, d):
     bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)
 
     write_upstream_config(confdir, upstream_config)
-    commit_config(confdir)
     update_build(upstream_config, confdir, setupdir, layerdir, d)
+    commit_config(confdir)
 
     bb.event.remove("bb.build.TaskProgress", None)
 

bitbake/lib/bb/tests/setup.py

@@ -187,6 +187,12 @@ print("BBPATH is {{}}".format(os.environ["BBPATH"]))
         bb_conf_path = os.path.join(bb_build_path, 'conf')
         self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env')))
 
+        with open(os.path.join(setuppath, 'config', "sources-fixed-revisions.json")) as f:
+            sources_fixed_revisions = json.load(f)
+        self.assertTrue('test-repo' in sources_fixed_revisions['sources'].keys())
+        revision = self.git('rev-parse HEAD', cwd=self.testrepopath).strip()
+        self.assertEqual(revision, sources_fixed_revisions['sources']['test-repo']['git-remote']['rev'])
+
         if "oe-template" in bitbake_config:
             with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f:
                 self.assertEqual(f.read(), bitbake_config["oe-template"])