diff options
Diffstat (limited to 'external/poky/bitbake/lib/bb/persist_data.py')
| -rw-r--r-- | external/poky/bitbake/lib/bb/persist_data.py | 237 |
1 files changed, 162 insertions, 75 deletions
diff --git a/external/poky/bitbake/lib/bb/persist_data.py b/external/poky/bitbake/lib/bb/persist_data.py index bef70186..7357ab2d 100644 --- a/external/poky/bitbake/lib/bb/persist_data.py +++ b/external/poky/bitbake/lib/bb/persist_data.py @@ -8,18 +8,8 @@ currently, providing a key/value store accessed by 'domain'. # Copyright (C) 2007 Richard Purdie # Copyright (C) 2010 Chris Larson <chris_larson@mentor.com> # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import collections import logging @@ -29,6 +19,7 @@ import warnings from bb.compat import total_ordering from collections import Mapping import sqlite3 +import contextlib sqlversion = sqlite3.sqlite_version_info if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): @@ -36,84 +27,184 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): logger = logging.getLogger("BitBake.PersistData") -if hasattr(sqlite3, 'enable_shared_cache'): - try: - sqlite3.enable_shared_cache(True) - except sqlite3.OperationalError: - pass - @total_ordering class SQLTable(collections.MutableMapping): + class _Decorators(object): + @staticmethod + def retry(*, reconnect=True): + """ + Decorator that restarts a function if a database locked sqlite + exception occurs. If reconnect is True, the database connection + will be closed and reopened each time a failure occurs + """ + def retry_wrapper(f): + def wrap_func(self, *args, **kwargs): + # Reconnect if necessary + if self.connection is None and reconnect: + self.reconnect() + + count = 0 + while True: + try: + return f(self, *args, **kwargs) + except sqlite3.OperationalError as exc: + if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)): + count = count + 1 + if reconnect: + self.reconnect() + continue + raise + return wrap_func + return retry_wrapper + + @staticmethod + def transaction(f): + """ + Decorator that starts a database transaction and creates a database + cursor for performing queries. If no exception is thrown, the + database results are commited. If an exception occurs, the database + is rolled back. In all cases, the cursor is closed after the + function ends. + + Note that the cursor is passed as an extra argument to the function + after `self` and before any of the normal arguments + """ + def wrap_func(self, *args, **kwargs): + # Context manager will COMMIT the database on success, + # or ROLLBACK on an exception + with self.connection: + # Automatically close the cursor when done + with contextlib.closing(self.connection.cursor()) as cursor: + return f(self, cursor, *args, **kwargs) + return wrap_func + """Object representing a table/domain in the database""" def __init__(self, cachefile, table): self.cachefile = cachefile self.table = table - self.cursor = connect(self.cachefile) - - self._execute("CREATE TABLE IF NOT EXISTS %s(key TEXT, value TEXT);" - % table) - - def _execute(self, *query): - """Execute a query, waiting to acquire a lock if necessary""" - count = 0 - while True: - try: - return self.cursor.execute(*query) - except sqlite3.OperationalError as exc: - if 'database is locked' in str(exc) and count < 500: - count = count + 1 + + self.connection = None + self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table) + + @_Decorators.retry(reconnect=False) + @_Decorators.transaction + def _setup_database(self, cursor): + cursor.execute("pragma synchronous = off;") + # Enable WAL and keep the autocheckpoint length small (the default is + # usually 1000). Persistent caches are usually read-mostly, so keeping + # this short will keep readers running quickly + cursor.execute("pragma journal_mode = WAL;") + cursor.execute("pragma wal_autocheckpoint = 100;") + + def reconnect(self): + if self.connection is not None: + self.connection.close() + self.connection = sqlite3.connect(self.cachefile, timeout=5) + self.connection.text_factory = str + self._setup_database() + + @_Decorators.retry() + @_Decorators.transaction + def _execute_single(self, cursor, *query): + """ + Executes a single query and discards the results. This correctly closes + the database cursor when finished + """ + cursor.execute(*query) + + @_Decorators.retry() + def _row_iter(self, f, *query): + """ + Helper function that returns a row iterator. Each time __next__ is + called on the iterator, the provided function is evaluated to determine + the return value + """ + class CursorIter(object): + def __init__(self, cursor): + self.cursor = cursor + + def __iter__(self): + return self + + def __next__(self): + row = self.cursor.fetchone() + if row is None: self.cursor.close() - self.cursor = connect(self.cachefile) - continue - raise + raise StopIteration + return f(row) + + def __enter__(self): + return self + + def __exit__(self, typ, value, traceback): + self.cursor.close() + return False + + cursor = self.connection.cursor() + try: + cursor.execute(*query) + return CursorIter(cursor) + except: + cursor.close() def __enter__(self): - self.cursor.__enter__() + self.connection.__enter__() return self def __exit__(self, *excinfo): - self.cursor.__exit__(*excinfo) - - def __getitem__(self, key): - data = self._execute("SELECT * from %s where key=?;" % - self.table, [key]) - for row in data: + self.connection.__exit__(*excinfo) + + @_Decorators.retry() + @_Decorators.transaction + def __getitem__(self, cursor, key): + cursor.execute("SELECT * from %s where key=?;" % self.table, [key]) + row = cursor.fetchone() + if row is not None: return row[1] raise KeyError(key) - def __delitem__(self, key): + @_Decorators.retry() + @_Decorators.transaction + def __delitem__(self, cursor, key): if key not in self: raise KeyError(key) - self._execute("DELETE from %s where key=?;" % self.table, [key]) + cursor.execute("DELETE from %s where key=?;" % self.table, [key]) - def __setitem__(self, key, value): + @_Decorators.retry() + @_Decorators.transaction + def __setitem__(self, cursor, key, value): if not isinstance(key, str): raise TypeError('Only string keys are supported') elif not isinstance(value, str): raise TypeError('Only string values are supported') - data = self._execute("SELECT * from %s where key=?;" % - self.table, [key]) - exists = len(list(data)) - if exists: - self._execute("UPDATE %s SET value=? WHERE key=?;" % self.table, - [value, key]) - else: - self._execute("INSERT into %s(key, value) values (?, ?);" % - self.table, [key, value]) - - def __contains__(self, key): - return key in set(self) + # Ensure the entire transaction (including SELECT) executes under write lock + cursor.execute("BEGIN EXCLUSIVE") - def __len__(self): - data = self._execute("SELECT COUNT(key) FROM %s;" % self.table) - for row in data: + cursor.execute("SELECT * from %s where key=?;" % self.table, [key]) + row = cursor.fetchone() + if row is not None: + cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key]) + else: + cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value]) + + @_Decorators.retry() + @_Decorators.transaction + def __contains__(self, cursor, key): + cursor.execute('SELECT * from %s where key=?;' % self.table, [key]) + return cursor.fetchone() is not None + + @_Decorators.retry() + @_Decorators.transaction + def __len__(self, cursor): + cursor.execute("SELECT COUNT(key) FROM %s;" % self.table) + row = cursor.fetchone() + if row is not None: return row[0] def __iter__(self): - data = self._execute("SELECT key FROM %s;" % self.table) - return (row[0] for row in data) + return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table) def __lt__(self, other): if not isinstance(other, Mapping): @@ -122,25 +213,27 @@ class SQLTable(collections.MutableMapping): return len(self) < len(other) def get_by_pattern(self, pattern): - data = self._execute("SELECT * FROM %s WHERE key LIKE ?;" % - self.table, [pattern]) - return [row[1] for row in data] + return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" % + self.table, [pattern]) def values(self): return list(self.itervalues()) def itervalues(self): - data = self._execute("SELECT value FROM %s;" % self.table) - return (row[0] for row in data) + return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" % + self.table) def items(self): return list(self.iteritems()) def iteritems(self): - return self._execute("SELECT * FROM %s;" % self.table) + return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" % + self.table) - def clear(self): - self._execute("DELETE FROM %s;" % self.table) + @_Decorators.retry() + @_Decorators.transaction + def clear(self, cursor): + cursor.execute("DELETE FROM %s;" % self.table) def has_key(self, key): return key in self @@ -194,12 +287,6 @@ class PersistData(object): """ del self.data[domain][key] -def connect(database): - connection = sqlite3.connect(database, timeout=5, isolation_level=None) - connection.execute("pragma synchronous = off;") - connection.text_factory = str - return connection - def persist(domain, d): """Convenience factory for SQLTable objects based upon metadata""" import bb.utils |