mirror of
git://git.yoctoproject.org/poky
synced 2026-10-07 12:45:22 +00:00
4cbce9cdf7
Introduce a PRSERVER_UPSTREAM variable that makes the local PR server connect to an "upstream" one. This makes it possible to implement local fixes to an upstream package (revision "x", in a way that gives the local update priority (revision "x.y"). Update the calculation of the new revisions to support the case when prior revisions are not integers, but have an "x.y..." format." Set the comments in the handle_get_pr() function in serv.py for details about the calculation of the local revision. This is done by going on supporting the "history" mode that wasn't used so far (revisions can return to a previous historical value), in addition to the default "no history" mode (revisions can never decrease). Rather than storing the history mode in the database table itself (i.e. "PRMAIN_hist" and "PRMAIN_nohist"), the history mode is now passed through the client requests. As a consequence, the table name is now "PRMAIN", which is incompatible with what was generated before, but avoids confusion if we kept the "PRMAIN_nohist" name for both "history" and "no history" modes. Update the server version to "2.0.0". (Bitbake rev: 48857ec3e075791bd73d92747c609a0a4fda0e0c) Signed-off-by: Michael Opdenacker <michael.opdenacker@bootlin.com> Cc: Joshua Watt <JPEWhacker@gmail.com> Cc: Tim Orling <ticotimo@gmail.com> Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
373 lines
14 KiB
Python
373 lines
14 KiB
Python
#
|
|
# Copyright BitBake Contributors
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import logging
|
|
import os.path
|
|
import errno
|
|
import prserv
|
|
import time
|
|
|
|
from . import increase_revision, revision_greater, revision_smaller
|
|
|
|
try:
|
|
import sqlite3
|
|
except ImportError:
|
|
from pysqlite2 import dbapi2 as sqlite3
|
|
|
|
logger = logging.getLogger("BitBake.PRserv")
|
|
|
|
sqlversion = sqlite3.sqlite_version_info
|
|
if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
|
|
raise Exception("sqlite3 version 3.3.0 or later is required.")
|
|
|
|
#
|
|
# "No History" mode - for a given query tuple (version, pkgarch, checksum),
|
|
# the returned value will be the largest among all the values of the same
|
|
# (version, pkgarch). This means the PR value returned can NOT be decremented.
|
|
#
|
|
# "History" mode - Return a new higher value for previously unseen query
|
|
# tuple (version, pkgarch, checksum), otherwise return historical value.
|
|
# Value can decrement if returning to a previous build.
|
|
#
|
|
|
|
class PRTable(object):
|
|
def __init__(self, conn, table, read_only):
|
|
self.conn = conn
|
|
self.read_only = read_only
|
|
self.dirty = False
|
|
self.table = table
|
|
|
|
if self.read_only:
|
|
table_exists = self._execute(
|
|
"SELECT count(*) FROM sqlite_master \
|
|
WHERE type='table' AND name='%s'" % (self.table))
|
|
if not table_exists:
|
|
raise prserv.NotFoundError
|
|
else:
|
|
self._execute("CREATE TABLE IF NOT EXISTS %s \
|
|
(version TEXT NOT NULL, \
|
|
pkgarch TEXT NOT NULL, \
|
|
checksum TEXT NOT NULL, \
|
|
value TEXT, \
|
|
PRIMARY KEY (version, pkgarch, checksum, value));" % self.table)
|
|
|
|
def _execute(self, *query):
|
|
"""Execute a query, waiting to acquire a lock if necessary"""
|
|
start = time.time()
|
|
end = start + 20
|
|
while True:
|
|
try:
|
|
return self.conn.execute(*query)
|
|
except sqlite3.OperationalError as exc:
|
|
if "is locked" in str(exc) and end > time.time():
|
|
continue
|
|
raise exc
|
|
|
|
def _extremum_value(self, rows, is_max):
|
|
value = None
|
|
|
|
for row in rows:
|
|
current_value = row[0]
|
|
if value is None:
|
|
value = current_value
|
|
else:
|
|
if is_max:
|
|
is_new_extremum = revision_greater(current_value, value)
|
|
else:
|
|
is_new_extremum = revision_smaller(current_value, value)
|
|
if is_new_extremum:
|
|
value = current_value
|
|
return value
|
|
|
|
def _max_value(self, rows):
|
|
return self._extremum_value(rows, True)
|
|
|
|
def _min_value(self, rows):
|
|
return self._extremum_value(rows, False)
|
|
|
|
def sync(self):
|
|
if not self.read_only:
|
|
self.conn.commit()
|
|
self._execute("BEGIN EXCLUSIVE TRANSACTION")
|
|
|
|
def sync_if_dirty(self):
|
|
if self.dirty:
|
|
self.sync()
|
|
self.dirty = False
|
|
|
|
def test_package(self, version, pkgarch):
|
|
"""Returns whether the specified package version is found in the database for the specified architecture"""
|
|
|
|
# Just returns the value if found or None otherwise
|
|
data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
|
|
(version, pkgarch))
|
|
row=data.fetchone()
|
|
if row is not None:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
def test_value(self, version, pkgarch, value):
|
|
"""Returns whether the specified value is found in the database for the specified package and architecture"""
|
|
|
|
# Just returns the value if found or None otherwise
|
|
data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
|
|
(version, pkgarch, value))
|
|
row=data.fetchone()
|
|
if row is not None:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def find_package_max_value(self, version, pkgarch):
|
|
"""Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
|
|
|
|
data = self._execute("SELECT value FROM %s where version=? AND pkgarch=?;" % (self.table),
|
|
(version, pkgarch))
|
|
rows = data.fetchall()
|
|
value = self._max_value(rows)
|
|
return value
|
|
|
|
def find_value(self, version, pkgarch, checksum, history=False):
|
|
"""Returns the value for the specified checksum if found or None otherwise."""
|
|
|
|
if history:
|
|
return self.find_min_value(version, pkgarch, checksum)
|
|
else:
|
|
return self.find_max_value(version, pkgarch, checksum)
|
|
|
|
|
|
def _find_extremum_value(self, version, pkgarch, checksum, is_max):
|
|
"""Returns the maximum (if is_max is True) or minimum (if is_max is False) value
|
|
for (version, pkgarch, checksum), or None if not found. Doesn't create a new value"""
|
|
|
|
data = self._execute("SELECT value FROM %s where version=? AND pkgarch=? AND checksum=?;" % (self.table),
|
|
(version, pkgarch, checksum))
|
|
rows = data.fetchall()
|
|
return self._extremum_value(rows, is_max)
|
|
|
|
def find_max_value(self, version, pkgarch, checksum):
|
|
return self._find_extremum_value(version, pkgarch, checksum, True)
|
|
|
|
def find_min_value(self, version, pkgarch, checksum):
|
|
return self._find_extremum_value(version, pkgarch, checksum, False)
|
|
|
|
def find_new_subvalue(self, version, pkgarch, base):
|
|
"""Take and increase the greatest "<base>.y" value for (version, pkgarch), or return "<base>.0" if not found.
|
|
This doesn't store a new value."""
|
|
|
|
data = self._execute("SELECT value FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base),
|
|
(version, pkgarch))
|
|
rows = data.fetchall()
|
|
value = self._max_value(rows)
|
|
|
|
if value is not None:
|
|
return increase_revision(value)
|
|
else:
|
|
return base + ".0"
|
|
|
|
def store_value(self, version, pkgarch, checksum, value):
|
|
"""Store new value in the database"""
|
|
|
|
try:
|
|
self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
|
|
(version, pkgarch, checksum, value))
|
|
except sqlite3.IntegrityError as exc:
|
|
logger.error(str(exc))
|
|
|
|
self.dirty = True
|
|
|
|
def _get_value(self, version, pkgarch, checksum, history):
|
|
|
|
max_value = self.find_package_max_value(version, pkgarch)
|
|
|
|
if max_value is None:
|
|
# version, pkgarch completely unknown. Return initial value.
|
|
return "0"
|
|
|
|
value = self.find_value(version, pkgarch, checksum, history)
|
|
|
|
if value is None:
|
|
# version, pkgarch found but not checksum. Create a new value from the maximum one
|
|
return increase_revision(max_value)
|
|
|
|
if history:
|
|
return value
|
|
|
|
# "no history" mode - If the value is not the maximum value for the package, need to increase it.
|
|
if max_value > value:
|
|
return increase_revision(max_value)
|
|
else:
|
|
return value
|
|
|
|
def get_value(self, version, pkgarch, checksum, history):
|
|
value = self._get_value(version, pkgarch, checksum, history)
|
|
if not self.read_only:
|
|
self.store_value(version, pkgarch, checksum, value)
|
|
return value
|
|
|
|
def _import_hist(self, version, pkgarch, checksum, value):
|
|
if self.read_only:
|
|
return None
|
|
|
|
val = None
|
|
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
|
|
(version, pkgarch, checksum))
|
|
row = data.fetchone()
|
|
if row is not None:
|
|
val=row[0]
|
|
else:
|
|
#no value found, try to insert
|
|
try:
|
|
self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
|
|
(version, pkgarch, checksum, value))
|
|
except sqlite3.IntegrityError as exc:
|
|
logger.error(str(exc))
|
|
|
|
self.dirty = True
|
|
|
|
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
|
|
(version, pkgarch, checksum))
|
|
row = data.fetchone()
|
|
if row is not None:
|
|
val = row[0]
|
|
return val
|
|
|
|
def _import_no_hist(self, version, pkgarch, checksum, value):
|
|
if self.read_only:
|
|
return None
|
|
|
|
try:
|
|
#try to insert
|
|
self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
|
|
(version, pkgarch, checksum, value))
|
|
except sqlite3.IntegrityError as exc:
|
|
#already have the record, try to update
|
|
try:
|
|
self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"
|
|
% (self.table),
|
|
(value, version, pkgarch, checksum, value))
|
|
except sqlite3.IntegrityError as exc:
|
|
logger.error(str(exc))
|
|
|
|
self.dirty = True
|
|
|
|
data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
|
|
(version, pkgarch, checksum, value))
|
|
row=data.fetchone()
|
|
if row is not None:
|
|
return row[0]
|
|
else:
|
|
return None
|
|
|
|
def importone(self, version, pkgarch, checksum, value, history=False):
|
|
if history:
|
|
return self._import_hist(version, pkgarch, checksum, value)
|
|
else:
|
|
return self._import_no_hist(version, pkgarch, checksum, value)
|
|
|
|
def export(self, version, pkgarch, checksum, colinfo, history=False):
|
|
metainfo = {}
|
|
#column info
|
|
if colinfo:
|
|
metainfo["tbl_name"] = self.table
|
|
metainfo["core_ver"] = prserv.__version__
|
|
metainfo["col_info"] = []
|
|
data = self._execute("PRAGMA table_info(%s);" % self.table)
|
|
for row in data:
|
|
col = {}
|
|
col["name"] = row["name"]
|
|
col["type"] = row["type"]
|
|
col["notnull"] = row["notnull"]
|
|
col["dflt_value"] = row["dflt_value"]
|
|
col["pk"] = row["pk"]
|
|
metainfo["col_info"].append(col)
|
|
|
|
#data info
|
|
datainfo = []
|
|
|
|
if history:
|
|
sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
|
|
else:
|
|
sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
|
|
(SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
|
|
WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
|
|
sqlarg = []
|
|
where = ""
|
|
if version:
|
|
where += "AND T1.version=? "
|
|
sqlarg.append(str(version))
|
|
if pkgarch:
|
|
where += "AND T1.pkgarch=? "
|
|
sqlarg.append(str(pkgarch))
|
|
if checksum:
|
|
where += "AND T1.checksum=? "
|
|
sqlarg.append(str(checksum))
|
|
|
|
sqlstmt += where + ";"
|
|
|
|
if len(sqlarg):
|
|
data = self._execute(sqlstmt, tuple(sqlarg))
|
|
else:
|
|
data = self._execute(sqlstmt)
|
|
for row in data:
|
|
if row["version"]:
|
|
col = {}
|
|
col["version"] = row["version"]
|
|
col["pkgarch"] = row["pkgarch"]
|
|
col["checksum"] = row["checksum"]
|
|
col["value"] = row["value"]
|
|
datainfo.append(col)
|
|
return (metainfo, datainfo)
|
|
|
|
def dump_db(self, fd):
|
|
writeCount = 0
|
|
for line in self.conn.iterdump():
|
|
writeCount = writeCount + len(line) + 1
|
|
fd.write(line)
|
|
fd.write("\n")
|
|
return writeCount
|
|
|
|
class PRData(object):
|
|
"""Object representing the PR database"""
|
|
def __init__(self, filename, read_only=False):
|
|
self.filename=os.path.abspath(filename)
|
|
self.read_only = read_only
|
|
#build directory hierarchy
|
|
try:
|
|
os.makedirs(os.path.dirname(self.filename))
|
|
except OSError as e:
|
|
if e.errno != errno.EEXIST:
|
|
raise e
|
|
uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "")
|
|
logger.debug("Opening PRServ database '%s'" % (uri))
|
|
self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False)
|
|
self.connection.row_factory=sqlite3.Row
|
|
if not self.read_only:
|
|
self.connection.execute("pragma synchronous = off;")
|
|
self.connection.execute("PRAGMA journal_mode = MEMORY;")
|
|
self._tables={}
|
|
|
|
def disconnect(self):
|
|
self.connection.close()
|
|
|
|
def __getitem__(self, tblname):
|
|
if not isinstance(tblname, str):
|
|
raise TypeError("tblname argument must be a string, not '%s'" %
|
|
type(tblname))
|
|
if tblname in self._tables:
|
|
return self._tables[tblname]
|
|
else:
|
|
tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.read_only)
|
|
return tableobj
|
|
|
|
def __delitem__(self, tblname):
|
|
if tblname in self._tables:
|
|
del self._tables[tblname]
|
|
logger.info("drop table %s" % (tblname))
|
|
self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
|