diff options
Diffstat (limited to 'migration')
48 files changed, 29553 insertions, 0 deletions
diff --git a/migration/block-dirty-bitmap.c b/migration/block-dirty-bitmap.c new file mode 100644 index 000000000..9aba7d9c2 --- /dev/null +++ b/migration/block-dirty-bitmap.c @@ -0,0 +1,1271 @@ +/* + * Block dirty bitmap postcopy migration + * + * Copyright IBM, Corp. 2009 + * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved. + * + * Authors: + * Liran Schour <lirans@il.ibm.com> + * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * This file is derived from migration/block.c, so it's author and IBM copyright + * are here, although content is quite different. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + * + * *** + * + * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable + * bitmaps are migrated. + * + * Bitmap migration implies creating bitmap with the same name and granularity + * in destination QEMU. If the bitmap with the same name (for the same node) + * already exists on destination an error will be generated. + * + * format of migration: + * + * # Header (shared for different chunk types) + * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags) + * [ 1 byte: node alias size ] \ flags & DEVICE_NAME + * [ n bytes: node alias ] / + * [ 1 byte: bitmap alias size ] \ flags & BITMAP_NAME + * [ n bytes: bitmap alias ] / + * + * # Start of bitmap migration (flags & START) + * header + * be64: granularity + * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap) + * bit 0 - bitmap is enabled + * bit 1 - bitmap is persistent + * bit 2 - bitmap is autoloading + * bits 3-7 - reserved, must be zero + * + * # Complete of bitmap migration (flags & COMPLETE) + * header + * + * # Data chunk of bitmap migration + * header + * be64: start sector + * be32: number of sectors + * [ be64: buffer size ] \ ! (flags & ZEROES) + * [ n bytes: buffer ] / + * + * The last chunk in stream should contain flags & EOS. The chunk may skip + * device and/or bitmap names, assuming them to be the same with the previous + * chunk. + */ + +#include "qemu/osdep.h" +#include "block/block.h" +#include "block/block_int.h" +#include "sysemu/block-backend.h" +#include "sysemu/runstate.h" +#include "qemu/main-loop.h" +#include "qemu/error-report.h" +#include "migration/misc.h" +#include "migration/migration.h" +#include "qemu-file.h" +#include "migration/vmstate.h" +#include "migration/register.h" +#include "qemu/hbitmap.h" +#include "qemu/cutils.h" +#include "qemu/id.h" +#include "qapi/error.h" +#include "qapi/qapi-commands-migration.h" +#include "qapi/qapi-visit-migration.h" +#include "qapi/clone-visitor.h" +#include "trace.h" + +#define CHUNK_SIZE (1 << 10) + +/* Flags occupy one, two or four bytes (Big Endian). The size is determined as + * follows: + * in first (most significant) byte bit 8 is clear --> one byte + * in first byte bit 8 is set --> two or four bytes, depending on second + * byte: + * | in second byte bit 8 is clear --> two bytes + * | in second byte bit 8 is set --> four bytes + */ +#define DIRTY_BITMAP_MIG_FLAG_EOS 0x01 +#define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02 +#define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04 +#define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08 +#define DIRTY_BITMAP_MIG_FLAG_START 0x10 +#define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20 +#define DIRTY_BITMAP_MIG_FLAG_BITS 0x40 + +#define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80 + +#define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01 +#define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02 +/* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */ +#define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8 + +/* State of one bitmap during save process */ +typedef struct SaveBitmapState { + /* Written during setup phase. */ + BlockDriverState *bs; + char *node_alias; + char *bitmap_alias; + BdrvDirtyBitmap *bitmap; + uint64_t total_sectors; + uint64_t sectors_per_chunk; + QSIMPLEQ_ENTRY(SaveBitmapState) entry; + uint8_t flags; + + /* For bulk phase. */ + bool bulk_completed; + uint64_t cur_sector; +} SaveBitmapState; + +/* State of the dirty bitmap migration (DBM) during save process */ +typedef struct DBMSaveState { + QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list; + + bool bulk_completed; + bool no_bitmaps; + + /* for send_bitmap_bits() */ + BlockDriverState *prev_bs; + BdrvDirtyBitmap *prev_bitmap; +} DBMSaveState; + +typedef struct LoadBitmapState { + BlockDriverState *bs; + BdrvDirtyBitmap *bitmap; + bool migrated; + bool enabled; +} LoadBitmapState; + +/* State of the dirty bitmap migration (DBM) during load process */ +typedef struct DBMLoadState { + uint32_t flags; + char node_alias[256]; + char bitmap_alias[256]; + char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1]; + BlockDriverState *bs; + BdrvDirtyBitmap *bitmap; + + bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */ + BitmapMigrationBitmapAlias *bmap_inner; + + /* + * cancelled + * Incoming migration is cancelled for some reason. That means that we + * still should read our chunks from migration stream, to not affect other + * migration objects (like RAM), but just ignore them and do not touch any + * bitmaps or nodes. + */ + bool cancelled; + + GSList *bitmaps; + QemuMutex lock; /* protect bitmaps */ +} DBMLoadState; + +typedef struct DBMState { + DBMSaveState save; + DBMLoadState load; +} DBMState; + +static DBMState dbm_state; + +/* For hash tables that map node/bitmap names to aliases */ +typedef struct AliasMapInnerNode { + char *string; + GHashTable *subtree; +} AliasMapInnerNode; + +static void free_alias_map_inner_node(void *amin_ptr) +{ + AliasMapInnerNode *amin = amin_ptr; + + g_free(amin->string); + g_hash_table_unref(amin->subtree); + g_free(amin); +} + +/** + * Construct an alias map based on the given QMP structure. + * + * (Note that we cannot store such maps in the MigrationParameters + * object, because that struct is defined by the QAPI schema, which + * makes it basically impossible to have dicts with arbitrary keys. + * Therefore, we instead have to construct these maps when migration + * starts.) + * + * @bbm is the block_bitmap_mapping from the migration parameters. + * + * If @name_to_alias is true, the returned hash table will map node + * and bitmap names to their respective aliases (for outgoing + * migration). + * + * If @name_to_alias is false, the returned hash table will map node + * and bitmap aliases to their respective names (for incoming + * migration). + * + * The hash table maps node names/aliases to AliasMapInnerNode + * objects, whose .string is the respective node alias/name, and whose + * .subtree table maps bitmap names/aliases to the respective bitmap + * alias/name. + */ +static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm, + bool name_to_alias, + Error **errp) +{ + GHashTable *alias_map; + size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1; + + alias_map = g_hash_table_new_full(g_str_hash, g_str_equal, + g_free, free_alias_map_inner_node); + + for (; bbm; bbm = bbm->next) { + const BitmapMigrationNodeAlias *bmna = bbm->value; + const BitmapMigrationBitmapAliasList *bmbal; + AliasMapInnerNode *amin; + GHashTable *bitmaps_map; + const char *node_map_from, *node_map_to; + GDestroyNotify gdn; + + if (!id_wellformed(bmna->alias)) { + error_setg(errp, "The node alias '%s' is not well-formed", + bmna->alias); + goto fail; + } + + if (strlen(bmna->alias) > UINT8_MAX) { + error_setg(errp, "The node alias '%s' is longer than %u bytes", + bmna->alias, UINT8_MAX); + goto fail; + } + + if (strlen(bmna->node_name) > max_node_name_len) { + error_setg(errp, "The node name '%s' is longer than %zu bytes", + bmna->node_name, max_node_name_len); + goto fail; + } + + if (name_to_alias) { + if (g_hash_table_contains(alias_map, bmna->node_name)) { + error_setg(errp, "The node name '%s' is mapped twice", + bmna->node_name); + goto fail; + } + + node_map_from = bmna->node_name; + node_map_to = bmna->alias; + } else { + if (g_hash_table_contains(alias_map, bmna->alias)) { + error_setg(errp, "The node alias '%s' is used twice", + bmna->alias); + goto fail; + } + + node_map_from = bmna->alias; + node_map_to = bmna->node_name; + } + + gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias; + bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, + gdn); + + amin = g_new(AliasMapInnerNode, 1); + *amin = (AliasMapInnerNode){ + .string = g_strdup(node_map_to), + .subtree = bitmaps_map, + }; + + g_hash_table_insert(alias_map, g_strdup(node_map_from), amin); + + for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) { + const BitmapMigrationBitmapAlias *bmba = bmbal->value; + const char *bmap_map_from; + + if (strlen(bmba->alias) > UINT8_MAX) { + error_setg(errp, + "The bitmap alias '%s' is longer than %u bytes", + bmba->alias, UINT8_MAX); + goto fail; + } + + if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) { + error_setg(errp, "The bitmap name '%s' is longer than %d bytes", + bmba->name, BDRV_BITMAP_MAX_NAME_SIZE); + goto fail; + } + + if (name_to_alias) { + bmap_map_from = bmba->name; + + if (g_hash_table_contains(bitmaps_map, bmba->name)) { + error_setg(errp, "The bitmap '%s'/'%s' is mapped twice", + bmna->node_name, bmba->name); + goto fail; + } + } else { + bmap_map_from = bmba->alias; + + if (g_hash_table_contains(bitmaps_map, bmba->alias)) { + error_setg(errp, "The bitmap alias '%s'/'%s' is used twice", + bmna->alias, bmba->alias); + goto fail; + } + } + + g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from), + QAPI_CLONE(BitmapMigrationBitmapAlias, bmba)); + } + } + + return alias_map; + +fail: + g_hash_table_destroy(alias_map); + return NULL; +} + +/** + * Run construct_alias_map() in both directions to check whether @bbm + * is valid. + * (This function is to be used by migration/migration.c to validate + * the user-specified block-bitmap-mapping migration parameter.) + * + * Returns true if and only if the mapping is valid. + */ +bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm, + Error **errp) +{ + GHashTable *alias_map; + + alias_map = construct_alias_map(bbm, true, errp); + if (!alias_map) { + return false; + } + g_hash_table_destroy(alias_map); + + alias_map = construct_alias_map(bbm, false, errp); + if (!alias_map) { + return false; + } + g_hash_table_destroy(alias_map); + + return true; +} + +static uint32_t qemu_get_bitmap_flags(QEMUFile *f) +{ + uint8_t flags = qemu_get_byte(f); + if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) { + flags = flags << 8 | qemu_get_byte(f); + if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) { + flags = flags << 16 | qemu_get_be16(f); + } + } + + return flags; +} + +static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags) +{ + /* The code currently does not send flags as more than one byte */ + assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS))); + + qemu_put_byte(f, flags); +} + +static void send_bitmap_header(QEMUFile *f, DBMSaveState *s, + SaveBitmapState *dbms, uint32_t additional_flags) +{ + BlockDriverState *bs = dbms->bs; + BdrvDirtyBitmap *bitmap = dbms->bitmap; + uint32_t flags = additional_flags; + trace_send_bitmap_header_enter(); + + if (bs != s->prev_bs) { + s->prev_bs = bs; + flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME; + } + + if (bitmap != s->prev_bitmap) { + s->prev_bitmap = bitmap; + flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME; + } + + qemu_put_bitmap_flags(f, flags); + + if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { + qemu_put_counted_string(f, dbms->node_alias); + } + + if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { + qemu_put_counted_string(f, dbms->bitmap_alias); + } +} + +static void send_bitmap_start(QEMUFile *f, DBMSaveState *s, + SaveBitmapState *dbms) +{ + send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START); + qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap)); + qemu_put_byte(f, dbms->flags); +} + +static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s, + SaveBitmapState *dbms) +{ + send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE); +} + +static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s, + SaveBitmapState *dbms, + uint64_t start_sector, uint32_t nr_sectors) +{ + /* align for buffer_is_zero() */ + uint64_t align = 4 * sizeof(long); + uint64_t unaligned_size = + bdrv_dirty_bitmap_serialization_size( + dbms->bitmap, start_sector << BDRV_SECTOR_BITS, + (uint64_t)nr_sectors << BDRV_SECTOR_BITS); + uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align); + uint8_t *buf = g_malloc0(buf_size); + uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS; + + bdrv_dirty_bitmap_serialize_part( + dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS, + (uint64_t)nr_sectors << BDRV_SECTOR_BITS); + + if (buffer_is_zero(buf, buf_size)) { + g_free(buf); + buf = NULL; + flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES; + } + + trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size); + + send_bitmap_header(f, s, dbms, flags); + + qemu_put_be64(f, start_sector); + qemu_put_be32(f, nr_sectors); + + /* if a block is zero we need to flush here since the network + * bandwidth is now a lot higher than the storage device bandwidth. + * thus if we queue zero blocks we slow down the migration. */ + if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) { + qemu_fflush(f); + } else { + qemu_put_be64(f, buf_size); + qemu_put_buffer(f, buf, buf_size); + } + + g_free(buf); +} + +/* Called with iothread lock taken. */ +static void dirty_bitmap_do_save_cleanup(DBMSaveState *s) +{ + SaveBitmapState *dbms; + + while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) { + QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry); + bdrv_dirty_bitmap_set_busy(dbms->bitmap, false); + bdrv_unref(dbms->bs); + g_free(dbms->node_alias); + g_free(dbms->bitmap_alias); + g_free(dbms); + } +} + +/* Called with iothread lock taken. */ +static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs, + const char *bs_name, GHashTable *alias_map) +{ + BdrvDirtyBitmap *bitmap; + SaveBitmapState *dbms; + GHashTable *bitmap_aliases; + const char *node_alias, *bitmap_name, *bitmap_alias; + Error *local_err = NULL; + + /* When an alias map is given, @bs_name must be @bs's node name */ + assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs))); + + FOR_EACH_DIRTY_BITMAP(bs, bitmap) { + if (bdrv_dirty_bitmap_name(bitmap)) { + break; + } + } + if (!bitmap) { + return 0; + } + + bitmap_name = bdrv_dirty_bitmap_name(bitmap); + + if (!bs_name || strcmp(bs_name, "") == 0) { + error_report("Bitmap '%s' in unnamed node can't be migrated", + bitmap_name); + return -1; + } + + if (alias_map) { + const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name); + + if (!amin) { + /* Skip bitmaps on nodes with no alias */ + return 0; + } + + node_alias = amin->string; + bitmap_aliases = amin->subtree; + } else { + node_alias = bs_name; + bitmap_aliases = NULL; + } + + if (node_alias[0] == '#') { + error_report("Bitmap '%s' in a node with auto-generated " + "name '%s' can't be migrated", + bitmap_name, node_alias); + return -1; + } + + FOR_EACH_DIRTY_BITMAP(bs, bitmap) { + BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL; + bitmap_name = bdrv_dirty_bitmap_name(bitmap); + if (!bitmap_name) { + continue; + } + + if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) { + error_report_err(local_err); + return -1; + } + + if (bitmap_aliases) { + BitmapMigrationBitmapAlias *bmap_inner; + + bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name); + if (!bmap_inner) { + /* Skip bitmaps with no alias */ + continue; + } + + bitmap_alias = bmap_inner->alias; + if (bmap_inner->has_transform) { + bitmap_transform = bmap_inner->transform; + } + } else { + if (strlen(bitmap_name) > UINT8_MAX) { + error_report("Cannot migrate bitmap '%s' on node '%s': " + "Name is longer than %u bytes", + bitmap_name, bs_name, UINT8_MAX); + return -1; + } + bitmap_alias = bitmap_name; + } + + bdrv_ref(bs); + bdrv_dirty_bitmap_set_busy(bitmap, true); + + dbms = g_new0(SaveBitmapState, 1); + dbms->bs = bs; + dbms->node_alias = g_strdup(node_alias); + dbms->bitmap_alias = g_strdup(bitmap_alias); + dbms->bitmap = bitmap; + dbms->total_sectors = bdrv_nb_sectors(bs); + dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU * + (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS); + assert(dbms->sectors_per_chunk != 0); + if (bdrv_dirty_bitmap_enabled(bitmap)) { + dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED; + } + if (bitmap_transform && + bitmap_transform->has_persistent) { + if (bitmap_transform->persistent) { + dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; + } + } else { + if (bdrv_dirty_bitmap_get_persistence(bitmap)) { + dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; + } + } + + QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry); + } + + return 0; +} + +/* Called with iothread lock taken. */ +static int init_dirty_bitmap_migration(DBMSaveState *s) +{ + BlockDriverState *bs; + SaveBitmapState *dbms; + GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL); + BlockBackend *blk; + const MigrationParameters *mig_params = &migrate_get_current()->parameters; + GHashTable *alias_map = NULL; + + if (mig_params->has_block_bitmap_mapping) { + alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true, + &error_abort); + } + + s->bulk_completed = false; + s->prev_bs = NULL; + s->prev_bitmap = NULL; + s->no_bitmaps = false; + + if (!alias_map) { + /* + * Use blockdevice name for direct (or filtered) children of named block + * backends. + */ + for (blk = blk_next(NULL); blk; blk = blk_next(blk)) { + const char *name = blk_name(blk); + + if (!name || strcmp(name, "") == 0) { + continue; + } + + bs = blk_bs(blk); + + /* Skip filters without bitmaps */ + while (bs && bs->drv && bs->drv->is_filter && + !bdrv_has_named_bitmaps(bs)) + { + bs = bdrv_filter_bs(bs); + } + + if (bs && bs->drv && !bs->drv->is_filter) { + if (add_bitmaps_to_list(s, bs, name, NULL)) { + goto fail; + } + g_hash_table_add(handled_by_blk, bs); + } + } + } + + for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) { + if (g_hash_table_contains(handled_by_blk, bs)) { + continue; + } + + if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) { + goto fail; + } + } + + /* unset migration flags here, to not roll back it */ + QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { + bdrv_dirty_bitmap_skip_store(dbms->bitmap, true); + } + + if (QSIMPLEQ_EMPTY(&s->dbms_list)) { + s->no_bitmaps = true; + } + + g_hash_table_destroy(handled_by_blk); + if (alias_map) { + g_hash_table_destroy(alias_map); + } + + return 0; + +fail: + g_hash_table_destroy(handled_by_blk); + if (alias_map) { + g_hash_table_destroy(alias_map); + } + dirty_bitmap_do_save_cleanup(s); + + return -1; +} + +/* Called with no lock taken. */ +static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s, + SaveBitmapState *dbms) +{ + uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector, + dbms->sectors_per_chunk); + + send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors); + + dbms->cur_sector += nr_sectors; + if (dbms->cur_sector >= dbms->total_sectors) { + dbms->bulk_completed = true; + } +} + +/* Called with no lock taken. */ +static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit) +{ + SaveBitmapState *dbms; + + QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { + while (!dbms->bulk_completed) { + bulk_phase_send_chunk(f, s, dbms); + if (limit && qemu_file_rate_limit(f)) { + return; + } + } + } + + s->bulk_completed = true; +} + +/* for SaveVMHandlers */ +static void dirty_bitmap_save_cleanup(void *opaque) +{ + DBMSaveState *s = &((DBMState *)opaque)->save; + + dirty_bitmap_do_save_cleanup(s); +} + +static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque) +{ + DBMSaveState *s = &((DBMState *)opaque)->save; + + trace_dirty_bitmap_save_iterate(migration_in_postcopy()); + + if (migration_in_postcopy() && !s->bulk_completed) { + bulk_phase(f, s, true); + } + + qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); + + return s->bulk_completed; +} + +/* Called with iothread lock taken. */ + +static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque) +{ + DBMSaveState *s = &((DBMState *)opaque)->save; + SaveBitmapState *dbms; + trace_dirty_bitmap_save_complete_enter(); + + if (!s->bulk_completed) { + bulk_phase(f, s, false); + } + + QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { + send_bitmap_complete(f, s, dbms); + } + + qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); + + trace_dirty_bitmap_save_complete_finish(); + + dirty_bitmap_save_cleanup(opaque); + return 0; +} + +static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque, + uint64_t max_size, + uint64_t *res_precopy_only, + uint64_t *res_compatible, + uint64_t *res_postcopy_only) +{ + DBMSaveState *s = &((DBMState *)opaque)->save; + SaveBitmapState *dbms; + uint64_t pending = 0; + + qemu_mutex_lock_iothread(); + + QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { + uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap); + uint64_t sectors = dbms->bulk_completed ? 0 : + dbms->total_sectors - dbms->cur_sector; + + pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran); + } + + qemu_mutex_unlock_iothread(); + + trace_dirty_bitmap_save_pending(pending, max_size); + + *res_postcopy_only += pending; +} + +/* First occurrence of this bitmap. It should be created if doesn't exist */ +static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s) +{ + Error *local_err = NULL; + uint32_t granularity = qemu_get_be32(f); + uint8_t flags = qemu_get_byte(f); + LoadBitmapState *b; + bool persistent; + + if (s->cancelled) { + return 0; + } + + if (s->bitmap) { + error_report("Bitmap with the same name ('%s') already exists on " + "destination", bdrv_dirty_bitmap_name(s->bitmap)); + return -EINVAL; + } else { + s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity, + s->bitmap_name, &local_err); + if (!s->bitmap) { + error_report_err(local_err); + return -EINVAL; + } + } + + if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) { + error_report("Unknown flags in migrated dirty bitmap header: %x", + flags); + return -EINVAL; + } + + if (s->bmap_inner && + s->bmap_inner->has_transform && + s->bmap_inner->transform->has_persistent) { + persistent = s->bmap_inner->transform->persistent; + } else { + persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT; + } + + if (persistent) { + bdrv_dirty_bitmap_set_persistence(s->bitmap, true); + } + + bdrv_disable_dirty_bitmap(s->bitmap); + if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) { + bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err); + if (local_err) { + error_report_err(local_err); + return -EINVAL; + } + } else { + bdrv_dirty_bitmap_set_busy(s->bitmap, true); + } + + b = g_new(LoadBitmapState, 1); + b->bs = s->bs; + b->bitmap = s->bitmap; + b->migrated = false; + b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED; + + s->bitmaps = g_slist_prepend(s->bitmaps, b); + + return 0; +} + +/* + * before_vm_start_handle_item + * + * g_slist_foreach helper + * + * item is LoadBitmapState* + * opaque is DBMLoadState* + */ +static void before_vm_start_handle_item(void *item, void *opaque) +{ + DBMLoadState *s = opaque; + LoadBitmapState *b = item; + + if (b->enabled) { + if (b->migrated) { + bdrv_enable_dirty_bitmap(b->bitmap); + } else { + bdrv_dirty_bitmap_enable_successor(b->bitmap); + } + } + + if (b->migrated) { + s->bitmaps = g_slist_remove(s->bitmaps, b); + g_free(b); + } +} + +void dirty_bitmap_mig_before_vm_start(void) +{ + DBMLoadState *s = &dbm_state.load; + qemu_mutex_lock(&s->lock); + + assert(!s->before_vm_start_handled); + g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s); + s->before_vm_start_handled = true; + + qemu_mutex_unlock(&s->lock); +} + +static void cancel_incoming_locked(DBMLoadState *s) +{ + GSList *item; + + if (s->cancelled) { + return; + } + + s->cancelled = true; + s->bs = NULL; + s->bitmap = NULL; + + /* Drop all unfinished bitmaps */ + for (item = s->bitmaps; item; item = g_slist_next(item)) { + LoadBitmapState *b = item->data; + + /* + * Bitmap must be unfinished, as finished bitmaps should already be + * removed from the list. + */ + assert(!s->before_vm_start_handled || !b->migrated); + if (bdrv_dirty_bitmap_has_successor(b->bitmap)) { + bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort); + } else { + bdrv_dirty_bitmap_set_busy(b->bitmap, false); + } + bdrv_release_dirty_bitmap(b->bitmap); + } + + g_slist_free_full(s->bitmaps, g_free); + s->bitmaps = NULL; +} + +void dirty_bitmap_mig_cancel_outgoing(void) +{ + dirty_bitmap_do_save_cleanup(&dbm_state.save); +} + +void dirty_bitmap_mig_cancel_incoming(void) +{ + DBMLoadState *s = &dbm_state.load; + + qemu_mutex_lock(&s->lock); + + cancel_incoming_locked(s); + + qemu_mutex_unlock(&s->lock); +} + +static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s) +{ + GSList *item; + trace_dirty_bitmap_load_complete(); + + if (s->cancelled) { + return; + } + + bdrv_dirty_bitmap_deserialize_finish(s->bitmap); + + if (bdrv_dirty_bitmap_has_successor(s->bitmap)) { + bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort); + } else { + bdrv_dirty_bitmap_set_busy(s->bitmap, false); + } + + for (item = s->bitmaps; item; item = g_slist_next(item)) { + LoadBitmapState *b = item->data; + + if (b->bitmap == s->bitmap) { + b->migrated = true; + if (s->before_vm_start_handled) { + s->bitmaps = g_slist_remove(s->bitmaps, b); + g_free(b); + } + break; + } + } +} + +static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s) +{ + uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS; + uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS; + trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS, + nr_bytes >> BDRV_SECTOR_BITS); + + if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) { + trace_dirty_bitmap_load_bits_zeroes(); + if (!s->cancelled) { + bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte, + nr_bytes, false); + } + } else { + size_t ret; + g_autofree uint8_t *buf = NULL; + uint64_t buf_size = qemu_get_be64(f); + uint64_t needed_size; + + /* + * The actual check for buf_size is done a bit later. We can't do it in + * cancelled mode as we don't have the bitmap to check the constraints + * (so, we allocate a buffer and read prior to the check). On the other + * hand, we shouldn't blindly g_malloc the number from the stream. + * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow + * a bit larger (which means that bitmap migration will fail anyway and + * the whole migration will most probably fail soon due to broken + * stream). + */ + if (buf_size > 10 * CHUNK_SIZE) { + error_report("Bitmap migration stream buffer allocation request " + "is too large"); + return -EIO; + } + + buf = g_malloc(buf_size); + ret = qemu_get_buffer(f, buf, buf_size); + if (ret != buf_size) { + error_report("Failed to read bitmap bits"); + return -EIO; + } + + if (s->cancelled) { + return 0; + } + + needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap, + first_byte, + nr_bytes); + + if (needed_size > buf_size || + buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long)) + /* Here used same alignment as in send_bitmap_bits */ + ) { + error_report("Migrated bitmap granularity doesn't " + "match the destination bitmap '%s' granularity", + bdrv_dirty_bitmap_name(s->bitmap)); + cancel_incoming_locked(s); + return 0; + } + + bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes, + false); + } + + return 0; +} + +static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s, + GHashTable *alias_map) +{ + GHashTable *bitmap_alias_map = NULL; + Error *local_err = NULL; + bool nothing; + s->flags = qemu_get_bitmap_flags(f); + trace_dirty_bitmap_load_header(s->flags); + + nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS); + + if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) { + if (!qemu_get_counted_string(f, s->node_alias)) { + error_report("Unable to read node alias string"); + return -EINVAL; + } + + if (!s->cancelled) { + if (alias_map) { + const AliasMapInnerNode *amin; + + amin = g_hash_table_lookup(alias_map, s->node_alias); + if (!amin) { + error_setg(&local_err, "Error: Unknown node alias '%s'", + s->node_alias); + s->bs = NULL; + } else { + bitmap_alias_map = amin->subtree; + s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err); + } + } else { + s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias, + &local_err); + } + if (!s->bs) { + error_report_err(local_err); + cancel_incoming_locked(s); + } + } + } else if (s->bs) { + if (alias_map) { + const AliasMapInnerNode *amin; + + /* Must be present in the map, or s->bs would not be set */ + amin = g_hash_table_lookup(alias_map, s->node_alias); + assert(amin != NULL); + + bitmap_alias_map = amin->subtree; + } + } else if (!nothing && !s->cancelled) { + error_report("Error: block device name is not set"); + cancel_incoming_locked(s); + } + + assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map); + + if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) { + const char *bitmap_name; + + if (!qemu_get_counted_string(f, s->bitmap_alias)) { + error_report("Unable to read bitmap alias string"); + return -EINVAL; + } + + bitmap_name = s->bitmap_alias; + if (!s->cancelled && bitmap_alias_map) { + BitmapMigrationBitmapAlias *bmap_inner; + + bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias); + if (!bmap_inner) { + error_report("Error: Unknown bitmap alias '%s' on node " + "'%s' (alias '%s')", s->bitmap_alias, + s->bs->node_name, s->node_alias); + cancel_incoming_locked(s); + } else { + bitmap_name = bmap_inner->name; + } + + s->bmap_inner = bmap_inner; + } + + if (!s->cancelled) { + g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name)); + s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name); + + /* + * bitmap may be NULL here, it wouldn't be an error if it is the + * first occurrence of the bitmap + */ + if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) { + error_report("Error: unknown dirty bitmap " + "'%s' for block device '%s'", + s->bitmap_name, s->bs->node_name); + cancel_incoming_locked(s); + } + } + } else if (!s->bitmap && !nothing && !s->cancelled) { + error_report("Error: block device name is not set"); + cancel_incoming_locked(s); + } + + return 0; +} + +/* + * dirty_bitmap_load + * + * Load sequence of dirty bitmap chunks. Return error only on fatal io stream + * violations. On other errors just cancel bitmaps incoming migration and return + * 0. + * + * Note, than when incoming bitmap migration is canceled, we still must read all + * our chunks (and just ignore them), to not affect other migration objects. + */ +static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id) +{ + GHashTable *alias_map = NULL; + const MigrationParameters *mig_params = &migrate_get_current()->parameters; + DBMLoadState *s = &((DBMState *)opaque)->load; + int ret = 0; + + trace_dirty_bitmap_load_enter(); + + if (version_id != 1) { + QEMU_LOCK_GUARD(&s->lock); + cancel_incoming_locked(s); + return -EINVAL; + } + + if (mig_params->has_block_bitmap_mapping) { + alias_map = construct_alias_map(mig_params->block_bitmap_mapping, + false, &error_abort); + } + + do { + QEMU_LOCK_GUARD(&s->lock); + + ret = dirty_bitmap_load_header(f, s, alias_map); + if (ret < 0) { + cancel_incoming_locked(s); + goto fail; + } + + if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) { + ret = dirty_bitmap_load_start(f, s); + } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) { + dirty_bitmap_load_complete(f, s); + } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) { + ret = dirty_bitmap_load_bits(f, s); + } + + if (!ret) { + ret = qemu_file_get_error(f); + } + + if (ret) { + cancel_incoming_locked(s); + goto fail; + } + } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS)); + + trace_dirty_bitmap_load_success(); + ret = 0; +fail: + if (alias_map) { + g_hash_table_destroy(alias_map); + } + return ret; +} + +static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque) +{ + DBMSaveState *s = &((DBMState *)opaque)->save; + SaveBitmapState *dbms = NULL; + + qemu_mutex_lock_iothread(); + if (init_dirty_bitmap_migration(s) < 0) { + qemu_mutex_unlock_iothread(); + return -1; + } + + QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) { + send_bitmap_start(f, s, dbms); + } + qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS); + qemu_mutex_unlock_iothread(); + return 0; +} + +static bool dirty_bitmap_is_active(void *opaque) +{ + DBMSaveState *s = &((DBMState *)opaque)->save; + + return migrate_dirty_bitmaps() && !s->no_bitmaps; +} + +static bool dirty_bitmap_is_active_iterate(void *opaque) +{ + return dirty_bitmap_is_active(opaque) && !runstate_is_running(); +} + +static bool dirty_bitmap_has_postcopy(void *opaque) +{ + return true; +} + +static SaveVMHandlers savevm_dirty_bitmap_handlers = { + .save_setup = dirty_bitmap_save_setup, + .save_live_complete_postcopy = dirty_bitmap_save_complete, + .save_live_complete_precopy = dirty_bitmap_save_complete, + .has_postcopy = dirty_bitmap_has_postcopy, + .save_live_pending = dirty_bitmap_save_pending, + .save_live_iterate = dirty_bitmap_save_iterate, + .is_active_iterate = dirty_bitmap_is_active_iterate, + .load_state = dirty_bitmap_load, + .save_cleanup = dirty_bitmap_save_cleanup, + .is_active = dirty_bitmap_is_active, +}; + +void dirty_bitmap_mig_init(void) +{ + QSIMPLEQ_INIT(&dbm_state.save.dbms_list); + qemu_mutex_init(&dbm_state.load.lock); + + register_savevm_live("dirty-bitmap", 0, 1, + &savevm_dirty_bitmap_handlers, + &dbm_state); +} diff --git a/migration/block.c b/migration/block.c new file mode 100644 index 000000000..a95097785 --- /dev/null +++ b/migration/block.c @@ -0,0 +1,1036 @@ +/* + * QEMU live block migration + * + * Copyright IBM, Corp. 2009 + * + * Authors: + * Liran Schour <lirans@il.ibm.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" +#include "qemu/cutils.h" +#include "qemu/queue.h" +#include "block.h" +#include "migration/misc.h" +#include "migration.h" +#include "migration/register.h" +#include "qemu-file.h" +#include "migration/vmstate.h" +#include "sysemu/block-backend.h" +#include "trace.h" + +#define BLK_MIG_BLOCK_SIZE (1 << 20) +#define BDRV_SECTORS_PER_DIRTY_CHUNK (BLK_MIG_BLOCK_SIZE >> BDRV_SECTOR_BITS) + +#define BLK_MIG_FLAG_DEVICE_BLOCK 0x01 +#define BLK_MIG_FLAG_EOS 0x02 +#define BLK_MIG_FLAG_PROGRESS 0x04 +#define BLK_MIG_FLAG_ZERO_BLOCK 0x08 + +#define MAX_IS_ALLOCATED_SEARCH (65536 * BDRV_SECTOR_SIZE) + +#define MAX_IO_BUFFERS 512 +#define MAX_PARALLEL_IO 16 + +/* #define DEBUG_BLK_MIGRATION */ + +#ifdef DEBUG_BLK_MIGRATION +#define DPRINTF(fmt, ...) \ + do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0) +#else +#define DPRINTF(fmt, ...) \ + do { } while (0) +#endif + +typedef struct BlkMigDevState { + /* Written during setup phase. Can be read without a lock. */ + BlockBackend *blk; + char *blk_name; + int shared_base; + int64_t total_sectors; + QSIMPLEQ_ENTRY(BlkMigDevState) entry; + Error *blocker; + + /* Only used by migration thread. Does not need a lock. */ + int bulk_completed; + int64_t cur_sector; + int64_t cur_dirty; + + /* Data in the aio_bitmap is protected by block migration lock. + * Allocation and free happen during setup and cleanup respectively. + */ + unsigned long *aio_bitmap; + + /* Protected by block migration lock. */ + int64_t completed_sectors; + + /* During migration this is protected by iothread lock / AioContext. + * Allocation and free happen during setup and cleanup respectively. + */ + BdrvDirtyBitmap *dirty_bitmap; +} BlkMigDevState; + +typedef struct BlkMigBlock { + /* Only used by migration thread. */ + uint8_t *buf; + BlkMigDevState *bmds; + int64_t sector; + int nr_sectors; + QEMUIOVector qiov; + BlockAIOCB *aiocb; + + /* Protected by block migration lock. */ + int ret; + QSIMPLEQ_ENTRY(BlkMigBlock) entry; +} BlkMigBlock; + +typedef struct BlkMigState { + QSIMPLEQ_HEAD(, BlkMigDevState) bmds_list; + int64_t total_sector_sum; + bool zero_blocks; + + /* Protected by lock. */ + QSIMPLEQ_HEAD(, BlkMigBlock) blk_list; + int submitted; + int read_done; + + /* Only used by migration thread. Does not need a lock. */ + int transferred; + int prev_progress; + int bulk_completed; + + /* Lock must be taken _inside_ the iothread lock and any AioContexts. */ + QemuMutex lock; +} BlkMigState; + +static BlkMigState block_mig_state; + +static void blk_mig_lock(void) +{ + qemu_mutex_lock(&block_mig_state.lock); +} + +static void blk_mig_unlock(void) +{ + qemu_mutex_unlock(&block_mig_state.lock); +} + +/* Must run outside of the iothread lock during the bulk phase, + * or the VM will stall. + */ + +static void blk_send(QEMUFile *f, BlkMigBlock * blk) +{ + int len; + uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK; + + if (block_mig_state.zero_blocks && + buffer_is_zero(blk->buf, BLK_MIG_BLOCK_SIZE)) { + flags |= BLK_MIG_FLAG_ZERO_BLOCK; + } + + /* sector number and flags */ + qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS) + | flags); + + /* device name */ + len = strlen(blk->bmds->blk_name); + qemu_put_byte(f, len); + qemu_put_buffer(f, (uint8_t *) blk->bmds->blk_name, len); + + /* if a block is zero we need to flush here since the network + * bandwidth is now a lot higher than the storage device bandwidth. + * thus if we queue zero blocks we slow down the migration */ + if (flags & BLK_MIG_FLAG_ZERO_BLOCK) { + qemu_fflush(f); + return; + } + + qemu_put_buffer(f, blk->buf, BLK_MIG_BLOCK_SIZE); +} + +int blk_mig_active(void) +{ + return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list); +} + +int blk_mig_bulk_active(void) +{ + return blk_mig_active() && !block_mig_state.bulk_completed; +} + +uint64_t blk_mig_bytes_transferred(void) +{ + BlkMigDevState *bmds; + uint64_t sum = 0; + + blk_mig_lock(); + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + sum += bmds->completed_sectors; + } + blk_mig_unlock(); + return sum << BDRV_SECTOR_BITS; +} + +uint64_t blk_mig_bytes_remaining(void) +{ + return blk_mig_bytes_total() - blk_mig_bytes_transferred(); +} + +uint64_t blk_mig_bytes_total(void) +{ + BlkMigDevState *bmds; + uint64_t sum = 0; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + sum += bmds->total_sectors; + } + return sum << BDRV_SECTOR_BITS; +} + + +/* Called with migration lock held. */ + +static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector) +{ + int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK; + + if (sector < blk_nb_sectors(bmds->blk)) { + return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] & + (1UL << (chunk % (sizeof(unsigned long) * 8)))); + } else { + return 0; + } +} + +/* Called with migration lock held. */ + +static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num, + int nb_sectors, int set) +{ + int64_t start, end; + unsigned long val, idx, bit; + + start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK; + end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK; + + for (; start <= end; start++) { + idx = start / (sizeof(unsigned long) * 8); + bit = start % (sizeof(unsigned long) * 8); + val = bmds->aio_bitmap[idx]; + if (set) { + val |= 1UL << bit; + } else { + val &= ~(1UL << bit); + } + bmds->aio_bitmap[idx] = val; + } +} + +static void alloc_aio_bitmap(BlkMigDevState *bmds) +{ + BlockBackend *bb = bmds->blk; + int64_t bitmap_size; + + bitmap_size = blk_nb_sectors(bb) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1; + bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8; + + bmds->aio_bitmap = g_malloc0(bitmap_size); +} + +/* Never hold migration lock when yielding to the main loop! */ + +static void blk_mig_read_cb(void *opaque, int ret) +{ + BlkMigBlock *blk = opaque; + + blk_mig_lock(); + blk->ret = ret; + + QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry); + bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0); + + block_mig_state.submitted--; + block_mig_state.read_done++; + assert(block_mig_state.submitted >= 0); + blk_mig_unlock(); +} + +/* Called with no lock taken. */ + +static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds) +{ + int64_t total_sectors = bmds->total_sectors; + int64_t cur_sector = bmds->cur_sector; + BlockBackend *bb = bmds->blk; + BlkMigBlock *blk; + int nr_sectors; + int64_t count; + + if (bmds->shared_base) { + qemu_mutex_lock_iothread(); + aio_context_acquire(blk_get_aio_context(bb)); + /* Skip unallocated sectors; intentionally treats failure or + * partial sector as an allocated sector */ + while (cur_sector < total_sectors && + !bdrv_is_allocated(blk_bs(bb), cur_sector * BDRV_SECTOR_SIZE, + MAX_IS_ALLOCATED_SEARCH, &count)) { + if (count < BDRV_SECTOR_SIZE) { + break; + } + cur_sector += count >> BDRV_SECTOR_BITS; + } + aio_context_release(blk_get_aio_context(bb)); + qemu_mutex_unlock_iothread(); + } + + if (cur_sector >= total_sectors) { + bmds->cur_sector = bmds->completed_sectors = total_sectors; + return 1; + } + + bmds->completed_sectors = cur_sector; + + cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1); + + /* we are going to transfer a full block even if it is not allocated */ + nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK; + + if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) { + nr_sectors = total_sectors - cur_sector; + } + + blk = g_new(BlkMigBlock, 1); + blk->buf = g_malloc(BLK_MIG_BLOCK_SIZE); + blk->bmds = bmds; + blk->sector = cur_sector; + blk->nr_sectors = nr_sectors; + + qemu_iovec_init_buf(&blk->qiov, blk->buf, nr_sectors * BDRV_SECTOR_SIZE); + + blk_mig_lock(); + block_mig_state.submitted++; + blk_mig_unlock(); + + /* We do not know if bs is under the main thread (and thus does + * not acquire the AioContext when doing AIO) or rather under + * dataplane. Thus acquire both the iothread mutex and the + * AioContext. + * + * This is ugly and will disappear when we make bdrv_* thread-safe, + * without the need to acquire the AioContext. + */ + qemu_mutex_lock_iothread(); + aio_context_acquire(blk_get_aio_context(bmds->blk)); + bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector * BDRV_SECTOR_SIZE, + nr_sectors * BDRV_SECTOR_SIZE); + blk->aiocb = blk_aio_preadv(bb, cur_sector * BDRV_SECTOR_SIZE, &blk->qiov, + 0, blk_mig_read_cb, blk); + aio_context_release(blk_get_aio_context(bmds->blk)); + qemu_mutex_unlock_iothread(); + + bmds->cur_sector = cur_sector + nr_sectors; + return (bmds->cur_sector >= total_sectors); +} + +/* Called with iothread lock taken. */ + +static int set_dirty_tracking(void) +{ + BlkMigDevState *bmds; + int ret; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + bmds->dirty_bitmap = bdrv_create_dirty_bitmap(blk_bs(bmds->blk), + BLK_MIG_BLOCK_SIZE, + NULL, NULL); + if (!bmds->dirty_bitmap) { + ret = -errno; + goto fail; + } + } + return 0; + +fail: + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + if (bmds->dirty_bitmap) { + bdrv_release_dirty_bitmap(bmds->dirty_bitmap); + } + } + return ret; +} + +/* Called with iothread lock taken. */ + +static void unset_dirty_tracking(void) +{ + BlkMigDevState *bmds; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + bdrv_release_dirty_bitmap(bmds->dirty_bitmap); + } +} + +static int init_blk_migration(QEMUFile *f) +{ + BlockDriverState *bs; + BlkMigDevState *bmds; + int64_t sectors; + BdrvNextIterator it; + int i, num_bs = 0; + struct { + BlkMigDevState *bmds; + BlockDriverState *bs; + } *bmds_bs; + Error *local_err = NULL; + int ret; + + block_mig_state.submitted = 0; + block_mig_state.read_done = 0; + block_mig_state.transferred = 0; + block_mig_state.total_sector_sum = 0; + block_mig_state.prev_progress = -1; + block_mig_state.bulk_completed = 0; + block_mig_state.zero_blocks = migrate_zero_blocks(); + + for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) { + num_bs++; + } + bmds_bs = g_malloc0(num_bs * sizeof(*bmds_bs)); + + for (i = 0, bs = bdrv_first(&it); bs; bs = bdrv_next(&it), i++) { + if (bdrv_is_read_only(bs)) { + continue; + } + + sectors = bdrv_nb_sectors(bs); + if (sectors <= 0) { + ret = sectors; + bdrv_next_cleanup(&it); + goto out; + } + + bmds = g_new0(BlkMigDevState, 1); + bmds->blk = blk_new(qemu_get_aio_context(), + BLK_PERM_CONSISTENT_READ, BLK_PERM_ALL); + bmds->blk_name = g_strdup(bdrv_get_device_name(bs)); + bmds->bulk_completed = 0; + bmds->total_sectors = sectors; + bmds->completed_sectors = 0; + bmds->shared_base = migrate_use_block_incremental(); + + assert(i < num_bs); + bmds_bs[i].bmds = bmds; + bmds_bs[i].bs = bs; + + block_mig_state.total_sector_sum += sectors; + + if (bmds->shared_base) { + trace_migration_block_init_shared(bdrv_get_device_name(bs)); + } else { + trace_migration_block_init_full(bdrv_get_device_name(bs)); + } + + QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry); + } + + /* Can only insert new BDSes now because doing so while iterating block + * devices may end up in a deadlock (iterating the new BDSes, too). */ + for (i = 0; i < num_bs; i++) { + BlkMigDevState *bmds = bmds_bs[i].bmds; + BlockDriverState *bs = bmds_bs[i].bs; + + if (bmds) { + ret = blk_insert_bs(bmds->blk, bs, &local_err); + if (ret < 0) { + error_report_err(local_err); + goto out; + } + + alloc_aio_bitmap(bmds); + error_setg(&bmds->blocker, "block device is in use by migration"); + bdrv_op_block_all(bs, bmds->blocker); + } + } + + ret = 0; +out: + g_free(bmds_bs); + return ret; +} + +/* Called with no lock taken. */ + +static int blk_mig_save_bulked_block(QEMUFile *f) +{ + int64_t completed_sector_sum = 0; + BlkMigDevState *bmds; + int progress; + int ret = 0; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + if (bmds->bulk_completed == 0) { + if (mig_save_device_bulk(f, bmds) == 1) { + /* completed bulk section for this device */ + bmds->bulk_completed = 1; + } + completed_sector_sum += bmds->completed_sectors; + ret = 1; + break; + } else { + completed_sector_sum += bmds->completed_sectors; + } + } + + if (block_mig_state.total_sector_sum != 0) { + progress = completed_sector_sum * 100 / + block_mig_state.total_sector_sum; + } else { + progress = 100; + } + if (progress != block_mig_state.prev_progress) { + block_mig_state.prev_progress = progress; + qemu_put_be64(f, (progress << BDRV_SECTOR_BITS) + | BLK_MIG_FLAG_PROGRESS); + DPRINTF("Completed %d %%\r", progress); + } + + return ret; +} + +static void blk_mig_reset_dirty_cursor(void) +{ + BlkMigDevState *bmds; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + bmds->cur_dirty = 0; + } +} + +/* Called with iothread lock and AioContext taken. */ + +static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds, + int is_async) +{ + BlkMigBlock *blk; + int64_t total_sectors = bmds->total_sectors; + int64_t sector; + int nr_sectors; + int ret = -EIO; + + for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) { + blk_mig_lock(); + if (bmds_aio_inflight(bmds, sector)) { + blk_mig_unlock(); + blk_drain(bmds->blk); + } else { + blk_mig_unlock(); + } + bdrv_dirty_bitmap_lock(bmds->dirty_bitmap); + if (bdrv_dirty_bitmap_get_locked(bmds->dirty_bitmap, + sector * BDRV_SECTOR_SIZE)) { + if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) { + nr_sectors = total_sectors - sector; + } else { + nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK; + } + bdrv_reset_dirty_bitmap_locked(bmds->dirty_bitmap, + sector * BDRV_SECTOR_SIZE, + nr_sectors * BDRV_SECTOR_SIZE); + bdrv_dirty_bitmap_unlock(bmds->dirty_bitmap); + + blk = g_new(BlkMigBlock, 1); + blk->buf = g_malloc(BLK_MIG_BLOCK_SIZE); + blk->bmds = bmds; + blk->sector = sector; + blk->nr_sectors = nr_sectors; + + if (is_async) { + qemu_iovec_init_buf(&blk->qiov, blk->buf, + nr_sectors * BDRV_SECTOR_SIZE); + + blk->aiocb = blk_aio_preadv(bmds->blk, + sector * BDRV_SECTOR_SIZE, + &blk->qiov, 0, blk_mig_read_cb, + blk); + + blk_mig_lock(); + block_mig_state.submitted++; + bmds_set_aio_inflight(bmds, sector, nr_sectors, 1); + blk_mig_unlock(); + } else { + ret = blk_pread(bmds->blk, sector * BDRV_SECTOR_SIZE, blk->buf, + nr_sectors * BDRV_SECTOR_SIZE); + if (ret < 0) { + goto error; + } + blk_send(f, blk); + + g_free(blk->buf); + g_free(blk); + } + + sector += nr_sectors; + bmds->cur_dirty = sector; + break; + } + + bdrv_dirty_bitmap_unlock(bmds->dirty_bitmap); + sector += BDRV_SECTORS_PER_DIRTY_CHUNK; + bmds->cur_dirty = sector; + } + + return (bmds->cur_dirty >= bmds->total_sectors); + +error: + trace_migration_block_save_device_dirty(sector); + g_free(blk->buf); + g_free(blk); + return ret; +} + +/* Called with iothread lock taken. + * + * return value: + * 0: too much data for max_downtime + * 1: few enough data for max_downtime +*/ +static int blk_mig_save_dirty_block(QEMUFile *f, int is_async) +{ + BlkMigDevState *bmds; + int ret = 1; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + aio_context_acquire(blk_get_aio_context(bmds->blk)); + ret = mig_save_device_dirty(f, bmds, is_async); + aio_context_release(blk_get_aio_context(bmds->blk)); + if (ret <= 0) { + break; + } + } + + return ret; +} + +/* Called with no locks taken. */ + +static int flush_blks(QEMUFile *f) +{ + BlkMigBlock *blk; + int ret = 0; + + trace_migration_block_flush_blks("Enter", block_mig_state.submitted, + block_mig_state.read_done, + block_mig_state.transferred); + + blk_mig_lock(); + while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) { + if (qemu_file_rate_limit(f)) { + break; + } + if (blk->ret < 0) { + ret = blk->ret; + break; + } + + QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry); + blk_mig_unlock(); + blk_send(f, blk); + blk_mig_lock(); + + g_free(blk->buf); + g_free(blk); + + block_mig_state.read_done--; + block_mig_state.transferred++; + assert(block_mig_state.read_done >= 0); + } + blk_mig_unlock(); + + trace_migration_block_flush_blks("Exit", block_mig_state.submitted, + block_mig_state.read_done, + block_mig_state.transferred); + return ret; +} + +/* Called with iothread lock taken. */ + +static int64_t get_remaining_dirty(void) +{ + BlkMigDevState *bmds; + int64_t dirty = 0; + + QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + aio_context_acquire(blk_get_aio_context(bmds->blk)); + dirty += bdrv_get_dirty_count(bmds->dirty_bitmap); + aio_context_release(blk_get_aio_context(bmds->blk)); + } + + return dirty; +} + + + +/* Called with iothread lock taken. */ +static void block_migration_cleanup_bmds(void) +{ + BlkMigDevState *bmds; + AioContext *ctx; + + unset_dirty_tracking(); + + while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) { + QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry); + bdrv_op_unblock_all(blk_bs(bmds->blk), bmds->blocker); + error_free(bmds->blocker); + + /* Save ctx, because bmds->blk can disappear during blk_unref. */ + ctx = blk_get_aio_context(bmds->blk); + aio_context_acquire(ctx); + blk_unref(bmds->blk); + aio_context_release(ctx); + + g_free(bmds->blk_name); + g_free(bmds->aio_bitmap); + g_free(bmds); + } +} + +/* Called with iothread lock taken. */ +static void block_migration_cleanup(void *opaque) +{ + BlkMigBlock *blk; + + bdrv_drain_all(); + + block_migration_cleanup_bmds(); + + blk_mig_lock(); + while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) { + QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry); + g_free(blk->buf); + g_free(blk); + } + blk_mig_unlock(); +} + +static int block_save_setup(QEMUFile *f, void *opaque) +{ + int ret; + + trace_migration_block_save("setup", block_mig_state.submitted, + block_mig_state.transferred); + + qemu_mutex_lock_iothread(); + ret = init_blk_migration(f); + if (ret < 0) { + qemu_mutex_unlock_iothread(); + return ret; + } + + /* start track dirty blocks */ + ret = set_dirty_tracking(); + + qemu_mutex_unlock_iothread(); + + if (ret) { + return ret; + } + + ret = flush_blks(f); + blk_mig_reset_dirty_cursor(); + qemu_put_be64(f, BLK_MIG_FLAG_EOS); + + return ret; +} + +static int block_save_iterate(QEMUFile *f, void *opaque) +{ + int ret; + int64_t last_ftell = qemu_ftell(f); + int64_t delta_ftell; + + trace_migration_block_save("iterate", block_mig_state.submitted, + block_mig_state.transferred); + + ret = flush_blks(f); + if (ret) { + return ret; + } + + blk_mig_reset_dirty_cursor(); + + /* control the rate of transfer */ + blk_mig_lock(); + while (block_mig_state.read_done * BLK_MIG_BLOCK_SIZE < + qemu_file_get_rate_limit(f) && + block_mig_state.submitted < MAX_PARALLEL_IO && + (block_mig_state.submitted + block_mig_state.read_done) < + MAX_IO_BUFFERS) { + blk_mig_unlock(); + if (block_mig_state.bulk_completed == 0) { + /* first finish the bulk phase */ + if (blk_mig_save_bulked_block(f) == 0) { + /* finished saving bulk on all devices */ + block_mig_state.bulk_completed = 1; + } + ret = 0; + } else { + /* Always called with iothread lock taken for + * simplicity, block_save_complete also calls it. + */ + qemu_mutex_lock_iothread(); + ret = blk_mig_save_dirty_block(f, 1); + qemu_mutex_unlock_iothread(); + } + if (ret < 0) { + return ret; + } + blk_mig_lock(); + if (ret != 0) { + /* no more dirty blocks */ + break; + } + } + blk_mig_unlock(); + + ret = flush_blks(f); + if (ret) { + return ret; + } + + qemu_put_be64(f, BLK_MIG_FLAG_EOS); + delta_ftell = qemu_ftell(f) - last_ftell; + if (delta_ftell > 0) { + return 1; + } else if (delta_ftell < 0) { + return -1; + } else { + return 0; + } +} + +/* Called with iothread lock taken. */ + +static int block_save_complete(QEMUFile *f, void *opaque) +{ + int ret; + + trace_migration_block_save("complete", block_mig_state.submitted, + block_mig_state.transferred); + + ret = flush_blks(f); + if (ret) { + return ret; + } + + blk_mig_reset_dirty_cursor(); + + /* we know for sure that save bulk is completed and + all async read completed */ + blk_mig_lock(); + assert(block_mig_state.submitted == 0); + blk_mig_unlock(); + + do { + ret = blk_mig_save_dirty_block(f, 0); + if (ret < 0) { + return ret; + } + } while (ret == 0); + + /* report completion */ + qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS); + + trace_migration_block_save_complete(); + + qemu_put_be64(f, BLK_MIG_FLAG_EOS); + + /* Make sure that our BlockBackends are gone, so that the block driver + * nodes can be inactivated. */ + block_migration_cleanup_bmds(); + + return 0; +} + +static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size, + uint64_t *res_precopy_only, + uint64_t *res_compatible, + uint64_t *res_postcopy_only) +{ + /* Estimate pending number of bytes to send */ + uint64_t pending; + + qemu_mutex_lock_iothread(); + pending = get_remaining_dirty(); + qemu_mutex_unlock_iothread(); + + blk_mig_lock(); + pending += block_mig_state.submitted * BLK_MIG_BLOCK_SIZE + + block_mig_state.read_done * BLK_MIG_BLOCK_SIZE; + blk_mig_unlock(); + + /* Report at least one block pending during bulk phase */ + if (pending <= max_size && !block_mig_state.bulk_completed) { + pending = max_size + BLK_MIG_BLOCK_SIZE; + } + + trace_migration_block_save_pending(pending); + /* We don't do postcopy */ + *res_precopy_only += pending; +} + +static int block_load(QEMUFile *f, void *opaque, int version_id) +{ + static int banner_printed; + int len, flags; + char device_name[256]; + int64_t addr; + BlockBackend *blk, *blk_prev = NULL; + Error *local_err = NULL; + uint8_t *buf; + int64_t total_sectors = 0; + int nr_sectors; + int ret; + BlockDriverInfo bdi; + int cluster_size = BLK_MIG_BLOCK_SIZE; + + do { + addr = qemu_get_be64(f); + + flags = addr & (BDRV_SECTOR_SIZE - 1); + addr >>= BDRV_SECTOR_BITS; + + if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) { + /* get device name */ + len = qemu_get_byte(f); + qemu_get_buffer(f, (uint8_t *)device_name, len); + device_name[len] = '\0'; + + blk = blk_by_name(device_name); + if (!blk) { + fprintf(stderr, "Error unknown block device %s\n", + device_name); + return -EINVAL; + } + + if (blk != blk_prev) { + blk_prev = blk; + total_sectors = blk_nb_sectors(blk); + if (total_sectors <= 0) { + error_report("Error getting length of block device %s", + device_name); + return -EINVAL; + } + + blk_invalidate_cache(blk, &local_err); + if (local_err) { + error_report_err(local_err); + return -EINVAL; + } + + ret = bdrv_get_info(blk_bs(blk), &bdi); + if (ret == 0 && bdi.cluster_size > 0 && + bdi.cluster_size <= BLK_MIG_BLOCK_SIZE && + BLK_MIG_BLOCK_SIZE % bdi.cluster_size == 0) { + cluster_size = bdi.cluster_size; + } else { + cluster_size = BLK_MIG_BLOCK_SIZE; + } + } + + if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) { + nr_sectors = total_sectors - addr; + } else { + nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK; + } + + if (flags & BLK_MIG_FLAG_ZERO_BLOCK) { + ret = blk_pwrite_zeroes(blk, addr * BDRV_SECTOR_SIZE, + nr_sectors * BDRV_SECTOR_SIZE, + BDRV_REQ_MAY_UNMAP); + } else { + int i; + int64_t cur_addr; + uint8_t *cur_buf; + + buf = g_malloc(BLK_MIG_BLOCK_SIZE); + qemu_get_buffer(f, buf, BLK_MIG_BLOCK_SIZE); + for (i = 0; i < BLK_MIG_BLOCK_SIZE / cluster_size; i++) { + cur_addr = addr * BDRV_SECTOR_SIZE + i * cluster_size; + cur_buf = buf + i * cluster_size; + + if ((!block_mig_state.zero_blocks || + cluster_size < BLK_MIG_BLOCK_SIZE) && + buffer_is_zero(cur_buf, cluster_size)) { + ret = blk_pwrite_zeroes(blk, cur_addr, + cluster_size, + BDRV_REQ_MAY_UNMAP); + } else { + ret = blk_pwrite(blk, cur_addr, cur_buf, + cluster_size, 0); + } + if (ret < 0) { + break; + } + } + g_free(buf); + } + + if (ret < 0) { + return ret; + } + } else if (flags & BLK_MIG_FLAG_PROGRESS) { + if (!banner_printed) { + printf("Receiving block device images\n"); + banner_printed = 1; + } + printf("Completed %d %%%c", (int)addr, + (addr == 100) ? '\n' : '\r'); + fflush(stdout); + } else if (!(flags & BLK_MIG_FLAG_EOS)) { + fprintf(stderr, "Unknown block migration flags: 0x%x\n", flags); + return -EINVAL; + } + ret = qemu_file_get_error(f); + if (ret != 0) { + return ret; + } + } while (!(flags & BLK_MIG_FLAG_EOS)); + + return 0; +} + +static bool block_is_active(void *opaque) +{ + return migrate_use_block(); +} + +static SaveVMHandlers savevm_block_handlers = { + .save_setup = block_save_setup, + .save_live_iterate = block_save_iterate, + .save_live_complete_precopy = block_save_complete, + .save_live_pending = block_save_pending, + .load_state = block_load, + .save_cleanup = block_migration_cleanup, + .is_active = block_is_active, +}; + +void blk_mig_init(void) +{ + QSIMPLEQ_INIT(&block_mig_state.bmds_list); + QSIMPLEQ_INIT(&block_mig_state.blk_list); + qemu_mutex_init(&block_mig_state.lock); + + register_savevm_live("block", 0, 1, &savevm_block_handlers, + &block_mig_state); +} diff --git a/migration/block.h b/migration/block.h new file mode 100644 index 000000000..3178609db --- /dev/null +++ b/migration/block.h @@ -0,0 +1,52 @@ +/* + * QEMU live block migration + * + * Copyright IBM, Corp. 2009 + * + * Authors: + * Liran Schour <lirans@il.ibm.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#ifndef MIGRATION_BLOCK_H +#define MIGRATION_BLOCK_H + +#ifdef CONFIG_LIVE_BLOCK_MIGRATION +int blk_mig_active(void); +int blk_mig_bulk_active(void); +uint64_t blk_mig_bytes_transferred(void); +uint64_t blk_mig_bytes_remaining(void); +uint64_t blk_mig_bytes_total(void); + +#else +static inline int blk_mig_active(void) +{ + return false; +} + +static inline int blk_mig_bulk_active(void) +{ + return false; +} + +static inline uint64_t blk_mig_bytes_transferred(void) +{ + return 0; +} + +static inline uint64_t blk_mig_bytes_remaining(void) +{ + return 0; +} + +static inline uint64_t blk_mig_bytes_total(void) +{ + return 0; +} +#endif /* CONFIG_LIVE_BLOCK_MIGRATION */ + +void migrate_set_block_enabled(bool value, Error **errp); +#endif /* MIGRATION_BLOCK_H */ diff --git a/migration/channel.c b/migration/channel.c new file mode 100644 index 000000000..c4fc000a1 --- /dev/null +++ b/migration/channel.c @@ -0,0 +1,101 @@ +/* + * QEMU live migration channel operations + * + * Copyright Red Hat, Inc. 2016 + * + * Authors: + * Daniel P. Berrange <berrange@redhat.com> + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "channel.h" +#include "tls.h" +#include "migration.h" +#include "qemu-file-channel.h" +#include "trace.h" +#include "qapi/error.h" +#include "io/channel-tls.h" +#include "io/channel-socket.h" +#include "qemu/yank.h" +#include "yank_functions.h" + +/** + * @migration_channel_process_incoming - Create new incoming migration channel + * + * Notice that TLS is special. For it we listen in a listener socket, + * and then create a new client socket from the TLS library. + * + * @ioc: Channel to which we are connecting + */ +void migration_channel_process_incoming(QIOChannel *ioc) +{ + MigrationState *s = migrate_get_current(); + Error *local_err = NULL; + + trace_migration_set_incoming_channel( + ioc, object_get_typename(OBJECT(ioc))); + + if (s->parameters.tls_creds && + *s->parameters.tls_creds && + !object_dynamic_cast(OBJECT(ioc), + TYPE_QIO_CHANNEL_TLS)) { + migration_tls_channel_process_incoming(s, ioc, &local_err); + } else { + migration_ioc_register_yank(ioc); + migration_ioc_process_incoming(ioc, &local_err); + } + + if (local_err) { + error_report_err(local_err); + } +} + + +/** + * @migration_channel_connect - Create new outgoing migration channel + * + * @s: Current migration state + * @ioc: Channel to which we are connecting + * @hostname: Where we want to connect + * @error: Error indicating failure to connect, free'd here + */ +void migration_channel_connect(MigrationState *s, + QIOChannel *ioc, + const char *hostname, + Error *error) +{ + trace_migration_set_outgoing_channel( + ioc, object_get_typename(OBJECT(ioc)), hostname, error); + + if (!error) { + if (s->parameters.tls_creds && + *s->parameters.tls_creds && + !object_dynamic_cast(OBJECT(ioc), + TYPE_QIO_CHANNEL_TLS)) { + migration_tls_channel_connect(s, ioc, hostname, &error); + + if (!error) { + /* tls_channel_connect will call back to this + * function after the TLS handshake, + * so we mustn't call migrate_fd_connect until then + */ + + return; + } + } else { + QEMUFile *f = qemu_fopen_channel_output(ioc); + + migration_ioc_register_yank(ioc); + + qemu_mutex_lock(&s->qemu_file_lock); + s->to_dst_file = f; + qemu_mutex_unlock(&s->qemu_file_lock); + } + } + migrate_fd_connect(s, error); + g_free(s->hostname); + error_free(error); +} diff --git a/migration/channel.h b/migration/channel.h new file mode 100644 index 000000000..67a461c28 --- /dev/null +++ b/migration/channel.h @@ -0,0 +1,27 @@ +/* + * QEMU live migration channel operations + * + * Copyright Red Hat, Inc. 2016 + * + * Authors: + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#ifndef QEMU_MIGRATION_CHANNEL_H +#define QEMU_MIGRATION_CHANNEL_H + +#include "io/channel.h" + +void migration_channel_process_incoming(QIOChannel *ioc); + +void migration_channel_connect(MigrationState *s, + QIOChannel *ioc, + const char *hostname, + Error *error_in); +#endif diff --git a/migration/colo-failover.c b/migration/colo-failover.c new file mode 100644 index 000000000..42453481c --- /dev/null +++ b/migration/colo-failover.c @@ -0,0 +1,86 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "migration/colo.h" +#include "migration/failover.h" +#include "qemu/main-loop.h" +#include "migration.h" +#include "qapi/error.h" +#include "qapi/qapi-commands-migration.h" +#include "qapi/qmp/qerror.h" +#include "qemu/error-report.h" +#include "trace.h" + +static QEMUBH *failover_bh; +static FailoverStatus failover_state; + +static void colo_failover_bh(void *opaque) +{ + int old_state; + + qemu_bh_delete(failover_bh); + failover_bh = NULL; + + old_state = failover_set_state(FAILOVER_STATUS_REQUIRE, + FAILOVER_STATUS_ACTIVE); + if (old_state != FAILOVER_STATUS_REQUIRE) { + error_report("Unknown error for failover, old_state = %s", + FailoverStatus_str(old_state)); + return; + } + + colo_do_failover(); +} + +void failover_request_active(Error **errp) +{ + if (failover_set_state(FAILOVER_STATUS_NONE, + FAILOVER_STATUS_REQUIRE) != FAILOVER_STATUS_NONE) { + error_setg(errp, "COLO failover is already activated"); + return; + } + failover_bh = qemu_bh_new(colo_failover_bh, NULL); + qemu_bh_schedule(failover_bh); +} + +void failover_init_state(void) +{ + failover_state = FAILOVER_STATUS_NONE; +} + +FailoverStatus failover_set_state(FailoverStatus old_state, + FailoverStatus new_state) +{ + FailoverStatus old; + + old = qatomic_cmpxchg(&failover_state, old_state, new_state); + if (old == old_state) { + trace_colo_failover_set_state(FailoverStatus_str(new_state)); + } + return old; +} + +FailoverStatus failover_get_state(void) +{ + return qatomic_read(&failover_state); +} + +void qmp_x_colo_lost_heartbeat(Error **errp) +{ + if (get_colo_mode() == COLO_MODE_NONE) { + error_setg(errp, QERR_FEATURE_DISABLED, "colo"); + return; + } + + failover_request_active(errp); +} diff --git a/migration/colo.c b/migration/colo.c new file mode 100644 index 000000000..241532526 --- /dev/null +++ b/migration/colo.c @@ -0,0 +1,927 @@ +/* + * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO) + * (a.k.a. Fault Tolerance or Continuous Replication) + * + * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. + * Copyright (c) 2016 FUJITSU LIMITED + * Copyright (c) 2016 Intel Corporation + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "sysemu/sysemu.h" +#include "qapi/error.h" +#include "qapi/qapi-commands-migration.h" +#include "qemu-file-channel.h" +#include "migration.h" +#include "qemu-file.h" +#include "savevm.h" +#include "migration/colo.h" +#include "block.h" +#include "io/channel-buffer.h" +#include "trace.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" +#include "qemu/rcu.h" +#include "migration/failover.h" +#include "migration/ram.h" +#ifdef CONFIG_REPLICATION +#include "block/replication.h" +#endif +#include "net/colo-compare.h" +#include "net/colo.h" +#include "block/block.h" +#include "qapi/qapi-events-migration.h" +#include "qapi/qmp/qerror.h" +#include "sysemu/cpus.h" +#include "sysemu/runstate.h" +#include "net/filter.h" + +static bool vmstate_loading; +static Notifier packets_compare_notifier; + +/* User need to know colo mode after COLO failover */ +static COLOMode last_colo_mode; + +#define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024) + +bool migration_in_colo_state(void) +{ + MigrationState *s = migrate_get_current(); + + return (s->state == MIGRATION_STATUS_COLO); +} + +bool migration_incoming_in_colo_state(void) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + return mis && (mis->state == MIGRATION_STATUS_COLO); +} + +static bool colo_runstate_is_stopped(void) +{ + return runstate_check(RUN_STATE_COLO) || !runstate_is_running(); +} + +static void secondary_vm_do_failover(void) +{ +/* COLO needs enable block-replication */ +#ifdef CONFIG_REPLICATION + int old_state; + MigrationIncomingState *mis = migration_incoming_get_current(); + Error *local_err = NULL; + + /* Can not do failover during the process of VM's loading VMstate, Or + * it will break the secondary VM. + */ + if (vmstate_loading) { + old_state = failover_set_state(FAILOVER_STATUS_ACTIVE, + FAILOVER_STATUS_RELAUNCH); + if (old_state != FAILOVER_STATUS_ACTIVE) { + error_report("Unknown error while do failover for secondary VM," + "old_state: %s", FailoverStatus_str(old_state)); + } + return; + } + + migrate_set_state(&mis->state, MIGRATION_STATUS_COLO, + MIGRATION_STATUS_COMPLETED); + + replication_stop_all(true, &local_err); + if (local_err) { + error_report_err(local_err); + local_err = NULL; + } + + /* Notify all filters of all NIC to do checkpoint */ + colo_notify_filters_event(COLO_EVENT_FAILOVER, &local_err); + if (local_err) { + error_report_err(local_err); + } + + if (!autostart) { + error_report("\"-S\" qemu option will be ignored in secondary side"); + /* recover runstate to normal migration finish state */ + autostart = true; + } + /* + * Make sure COLO incoming thread not block in recv or send, + * If mis->from_src_file and mis->to_src_file use the same fd, + * The second shutdown() will return -1, we ignore this value, + * It is harmless. + */ + if (mis->from_src_file) { + qemu_file_shutdown(mis->from_src_file); + } + if (mis->to_src_file) { + qemu_file_shutdown(mis->to_src_file); + } + + old_state = failover_set_state(FAILOVER_STATUS_ACTIVE, + FAILOVER_STATUS_COMPLETED); + if (old_state != FAILOVER_STATUS_ACTIVE) { + error_report("Incorrect state (%s) while doing failover for " + "secondary VM", FailoverStatus_str(old_state)); + return; + } + /* Notify COLO incoming thread that failover work is finished */ + qemu_sem_post(&mis->colo_incoming_sem); + + /* For Secondary VM, jump to incoming co */ + if (mis->migration_incoming_co) { + qemu_coroutine_enter(mis->migration_incoming_co); + } +#else + abort(); +#endif +} + +static void primary_vm_do_failover(void) +{ +#ifdef CONFIG_REPLICATION + MigrationState *s = migrate_get_current(); + int old_state; + Error *local_err = NULL; + + migrate_set_state(&s->state, MIGRATION_STATUS_COLO, + MIGRATION_STATUS_COMPLETED); + /* + * kick COLO thread which might wait at + * qemu_sem_wait(&s->colo_checkpoint_sem). + */ + colo_checkpoint_notify(s); + + /* + * Wake up COLO thread which may blocked in recv() or send(), + * The s->rp_state.from_dst_file and s->to_dst_file may use the + * same fd, but we still shutdown the fd for twice, it is harmless. + */ + if (s->to_dst_file) { + qemu_file_shutdown(s->to_dst_file); + } + if (s->rp_state.from_dst_file) { + qemu_file_shutdown(s->rp_state.from_dst_file); + } + + old_state = failover_set_state(FAILOVER_STATUS_ACTIVE, + FAILOVER_STATUS_COMPLETED); + if (old_state != FAILOVER_STATUS_ACTIVE) { + error_report("Incorrect state (%s) while doing failover for Primary VM", + FailoverStatus_str(old_state)); + return; + } + + replication_stop_all(true, &local_err); + if (local_err) { + error_report_err(local_err); + local_err = NULL; + } + + /* Notify COLO thread that failover work is finished */ + qemu_sem_post(&s->colo_exit_sem); +#else + abort(); +#endif +} + +COLOMode get_colo_mode(void) +{ + if (migration_in_colo_state()) { + return COLO_MODE_PRIMARY; + } else if (migration_incoming_in_colo_state()) { + return COLO_MODE_SECONDARY; + } else { + return COLO_MODE_NONE; + } +} + +void colo_do_failover(void) +{ + /* Make sure VM stopped while failover happened. */ + if (!colo_runstate_is_stopped()) { + vm_stop_force_state(RUN_STATE_COLO); + } + + switch (last_colo_mode = get_colo_mode()) { + case COLO_MODE_PRIMARY: + primary_vm_do_failover(); + break; + case COLO_MODE_SECONDARY: + secondary_vm_do_failover(); + break; + default: + error_report("colo_do_failover failed because the colo mode" + " could not be obtained"); + } +} + +#ifdef CONFIG_REPLICATION +void qmp_xen_set_replication(bool enable, bool primary, + bool has_failover, bool failover, + Error **errp) +{ + ReplicationMode mode = primary ? + REPLICATION_MODE_PRIMARY : + REPLICATION_MODE_SECONDARY; + + if (has_failover && enable) { + error_setg(errp, "Parameter 'failover' is only for" + " stopping replication"); + return; + } + + if (enable) { + replication_start_all(mode, errp); + } else { + if (!has_failover) { + failover = NULL; + } + replication_stop_all(failover, failover ? NULL : errp); + } +} + +ReplicationStatus *qmp_query_xen_replication_status(Error **errp) +{ + Error *err = NULL; + ReplicationStatus *s = g_new0(ReplicationStatus, 1); + + replication_get_error_all(&err); + if (err) { + s->error = true; + s->has_desc = true; + s->desc = g_strdup(error_get_pretty(err)); + } else { + s->error = false; + } + + error_free(err); + return s; +} + +void qmp_xen_colo_do_checkpoint(Error **errp) +{ + Error *err = NULL; + + replication_do_checkpoint_all(&err); + if (err) { + error_propagate(errp, err); + return; + } + /* Notify all filters of all NIC to do checkpoint */ + colo_notify_filters_event(COLO_EVENT_CHECKPOINT, errp); +} +#endif + +COLOStatus *qmp_query_colo_status(Error **errp) +{ + COLOStatus *s = g_new0(COLOStatus, 1); + + s->mode = get_colo_mode(); + s->last_mode = last_colo_mode; + + switch (failover_get_state()) { + case FAILOVER_STATUS_NONE: + s->reason = COLO_EXIT_REASON_NONE; + break; + case FAILOVER_STATUS_COMPLETED: + s->reason = COLO_EXIT_REASON_REQUEST; + break; + default: + if (migration_in_colo_state()) { + s->reason = COLO_EXIT_REASON_PROCESSING; + } else { + s->reason = COLO_EXIT_REASON_ERROR; + } + } + + return s; +} + +static void colo_send_message(QEMUFile *f, COLOMessage msg, + Error **errp) +{ + int ret; + + if (msg >= COLO_MESSAGE__MAX) { + error_setg(errp, "%s: Invalid message", __func__); + return; + } + qemu_put_be32(f, msg); + qemu_fflush(f); + + ret = qemu_file_get_error(f); + if (ret < 0) { + error_setg_errno(errp, -ret, "Can't send COLO message"); + } + trace_colo_send_message(COLOMessage_str(msg)); +} + +static void colo_send_message_value(QEMUFile *f, COLOMessage msg, + uint64_t value, Error **errp) +{ + Error *local_err = NULL; + int ret; + + colo_send_message(f, msg, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + qemu_put_be64(f, value); + qemu_fflush(f); + + ret = qemu_file_get_error(f); + if (ret < 0) { + error_setg_errno(errp, -ret, "Failed to send value for message:%s", + COLOMessage_str(msg)); + } +} + +static COLOMessage colo_receive_message(QEMUFile *f, Error **errp) +{ + COLOMessage msg; + int ret; + + msg = qemu_get_be32(f); + ret = qemu_file_get_error(f); + if (ret < 0) { + error_setg_errno(errp, -ret, "Can't receive COLO message"); + return msg; + } + if (msg >= COLO_MESSAGE__MAX) { + error_setg(errp, "%s: Invalid message", __func__); + return msg; + } + trace_colo_receive_message(COLOMessage_str(msg)); + return msg; +} + +static void colo_receive_check_message(QEMUFile *f, COLOMessage expect_msg, + Error **errp) +{ + COLOMessage msg; + Error *local_err = NULL; + + msg = colo_receive_message(f, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + if (msg != expect_msg) { + error_setg(errp, "Unexpected COLO message %d, expected %d", + msg, expect_msg); + } +} + +static uint64_t colo_receive_message_value(QEMUFile *f, uint32_t expect_msg, + Error **errp) +{ + Error *local_err = NULL; + uint64_t value; + int ret; + + colo_receive_check_message(f, expect_msg, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return 0; + } + + value = qemu_get_be64(f); + ret = qemu_file_get_error(f); + if (ret < 0) { + error_setg_errno(errp, -ret, "Failed to get value for COLO message: %s", + COLOMessage_str(expect_msg)); + } + return value; +} + +static int colo_do_checkpoint_transaction(MigrationState *s, + QIOChannelBuffer *bioc, + QEMUFile *fb) +{ + Error *local_err = NULL; + int ret = -1; + + colo_send_message(s->to_dst_file, COLO_MESSAGE_CHECKPOINT_REQUEST, + &local_err); + if (local_err) { + goto out; + } + + colo_receive_check_message(s->rp_state.from_dst_file, + COLO_MESSAGE_CHECKPOINT_REPLY, &local_err); + if (local_err) { + goto out; + } + /* Reset channel-buffer directly */ + qio_channel_io_seek(QIO_CHANNEL(bioc), 0, 0, NULL); + bioc->usage = 0; + + qemu_mutex_lock_iothread(); + if (failover_get_state() != FAILOVER_STATUS_NONE) { + qemu_mutex_unlock_iothread(); + goto out; + } + vm_stop_force_state(RUN_STATE_COLO); + qemu_mutex_unlock_iothread(); + trace_colo_vm_state_change("run", "stop"); + /* + * Failover request bh could be called after vm_stop_force_state(), + * So we need check failover_request_is_active() again. + */ + if (failover_get_state() != FAILOVER_STATUS_NONE) { + goto out; + } + qemu_mutex_lock_iothread(); + +#ifdef CONFIG_REPLICATION + replication_do_checkpoint_all(&local_err); + if (local_err) { + qemu_mutex_unlock_iothread(); + goto out; + } +#else + abort(); +#endif + + colo_send_message(s->to_dst_file, COLO_MESSAGE_VMSTATE_SEND, &local_err); + if (local_err) { + qemu_mutex_unlock_iothread(); + goto out; + } + /* Note: device state is saved into buffer */ + ret = qemu_save_device_state(fb); + + qemu_mutex_unlock_iothread(); + if (ret < 0) { + goto out; + } + + if (migrate_auto_converge()) { + mig_throttle_counter_reset(); + } + /* + * Only save VM's live state, which not including device state. + * TODO: We may need a timeout mechanism to prevent COLO process + * to be blocked here. + */ + qemu_savevm_live_state(s->to_dst_file); + + qemu_fflush(fb); + + /* + * We need the size of the VMstate data in Secondary side, + * With which we can decide how much data should be read. + */ + colo_send_message_value(s->to_dst_file, COLO_MESSAGE_VMSTATE_SIZE, + bioc->usage, &local_err); + if (local_err) { + goto out; + } + + qemu_put_buffer(s->to_dst_file, bioc->data, bioc->usage); + qemu_fflush(s->to_dst_file); + ret = qemu_file_get_error(s->to_dst_file); + if (ret < 0) { + goto out; + } + + colo_receive_check_message(s->rp_state.from_dst_file, + COLO_MESSAGE_VMSTATE_RECEIVED, &local_err); + if (local_err) { + goto out; + } + + qemu_event_reset(&s->colo_checkpoint_event); + colo_notify_compares_event(NULL, COLO_EVENT_CHECKPOINT, &local_err); + if (local_err) { + goto out; + } + + colo_receive_check_message(s->rp_state.from_dst_file, + COLO_MESSAGE_VMSTATE_LOADED, &local_err); + if (local_err) { + goto out; + } + + ret = 0; + + qemu_mutex_lock_iothread(); + vm_start(); + qemu_mutex_unlock_iothread(); + trace_colo_vm_state_change("stop", "run"); + +out: + if (local_err) { + error_report_err(local_err); + } + return ret; +} + +static void colo_compare_notify_checkpoint(Notifier *notifier, void *data) +{ + colo_checkpoint_notify(data); +} + +static void colo_process_checkpoint(MigrationState *s) +{ + QIOChannelBuffer *bioc; + QEMUFile *fb = NULL; + int64_t current_time = qemu_clock_get_ms(QEMU_CLOCK_HOST); + Error *local_err = NULL; + int ret; + + if (get_colo_mode() != COLO_MODE_PRIMARY) { + error_report("COLO mode must be COLO_MODE_PRIMARY"); + return; + } + + failover_init_state(); + + s->rp_state.from_dst_file = qemu_file_get_return_path(s->to_dst_file); + if (!s->rp_state.from_dst_file) { + error_report("Open QEMUFile from_dst_file failed"); + goto out; + } + + packets_compare_notifier.notify = colo_compare_notify_checkpoint; + colo_compare_register_notifier(&packets_compare_notifier); + + /* + * Wait for Secondary finish loading VM states and enter COLO + * restore. + */ + colo_receive_check_message(s->rp_state.from_dst_file, + COLO_MESSAGE_CHECKPOINT_READY, &local_err); + if (local_err) { + goto out; + } + bioc = qio_channel_buffer_new(COLO_BUFFER_BASE_SIZE); + fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc)); + object_unref(OBJECT(bioc)); + + qemu_mutex_lock_iothread(); +#ifdef CONFIG_REPLICATION + replication_start_all(REPLICATION_MODE_PRIMARY, &local_err); + if (local_err) { + qemu_mutex_unlock_iothread(); + goto out; + } +#else + abort(); +#endif + + vm_start(); + qemu_mutex_unlock_iothread(); + trace_colo_vm_state_change("stop", "run"); + + timer_mod(s->colo_delay_timer, + current_time + s->parameters.x_checkpoint_delay); + + while (s->state == MIGRATION_STATUS_COLO) { + if (failover_get_state() != FAILOVER_STATUS_NONE) { + error_report("failover request"); + goto out; + } + + qemu_event_wait(&s->colo_checkpoint_event); + + if (s->state != MIGRATION_STATUS_COLO) { + goto out; + } + ret = colo_do_checkpoint_transaction(s, bioc, fb); + if (ret < 0) { + goto out; + } + } + +out: + /* Throw the unreported error message after exited from loop */ + if (local_err) { + error_report_err(local_err); + } + + if (fb) { + qemu_fclose(fb); + } + + /* + * There are only two reasons we can get here, some error happened + * or the user triggered failover. + */ + switch (failover_get_state()) { + case FAILOVER_STATUS_COMPLETED: + qapi_event_send_colo_exit(COLO_MODE_PRIMARY, + COLO_EXIT_REASON_REQUEST); + break; + default: + qapi_event_send_colo_exit(COLO_MODE_PRIMARY, + COLO_EXIT_REASON_ERROR); + } + + /* Hope this not to be too long to wait here */ + qemu_sem_wait(&s->colo_exit_sem); + qemu_sem_destroy(&s->colo_exit_sem); + + /* + * It is safe to unregister notifier after failover finished. + * Besides, colo_delay_timer and colo_checkpoint_sem can't be + * released before unregister notifier, or there will be use-after-free + * error. + */ + colo_compare_unregister_notifier(&packets_compare_notifier); + timer_free(s->colo_delay_timer); + qemu_event_destroy(&s->colo_checkpoint_event); + + /* + * Must be called after failover BH is completed, + * Or the failover BH may shutdown the wrong fd that + * re-used by other threads after we release here. + */ + if (s->rp_state.from_dst_file) { + qemu_fclose(s->rp_state.from_dst_file); + s->rp_state.from_dst_file = NULL; + } +} + +void colo_checkpoint_notify(void *opaque) +{ + MigrationState *s = opaque; + int64_t next_notify_time; + + qemu_event_set(&s->colo_checkpoint_event); + s->colo_checkpoint_time = qemu_clock_get_ms(QEMU_CLOCK_HOST); + next_notify_time = s->colo_checkpoint_time + + s->parameters.x_checkpoint_delay; + timer_mod(s->colo_delay_timer, next_notify_time); +} + +void migrate_start_colo_process(MigrationState *s) +{ + qemu_mutex_unlock_iothread(); + qemu_event_init(&s->colo_checkpoint_event, false); + s->colo_delay_timer = timer_new_ms(QEMU_CLOCK_HOST, + colo_checkpoint_notify, s); + + qemu_sem_init(&s->colo_exit_sem, 0); + migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_COLO); + colo_process_checkpoint(s); + qemu_mutex_lock_iothread(); +} + +static void colo_incoming_process_checkpoint(MigrationIncomingState *mis, + QEMUFile *fb, QIOChannelBuffer *bioc, Error **errp) +{ + uint64_t total_size; + uint64_t value; + Error *local_err = NULL; + int ret; + + qemu_mutex_lock_iothread(); + vm_stop_force_state(RUN_STATE_COLO); + trace_colo_vm_state_change("run", "stop"); + qemu_mutex_unlock_iothread(); + + /* FIXME: This is unnecessary for periodic checkpoint mode */ + colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_REPLY, + &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + + colo_receive_check_message(mis->from_src_file, + COLO_MESSAGE_VMSTATE_SEND, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + + qemu_mutex_lock_iothread(); + cpu_synchronize_all_states(); + ret = qemu_loadvm_state_main(mis->from_src_file, mis); + qemu_mutex_unlock_iothread(); + + if (ret < 0) { + error_setg(errp, "Load VM's live state (ram) error"); + return; + } + + value = colo_receive_message_value(mis->from_src_file, + COLO_MESSAGE_VMSTATE_SIZE, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + + /* + * Read VM device state data into channel buffer, + * It's better to re-use the memory allocated. + * Here we need to handle the channel buffer directly. + */ + if (value > bioc->capacity) { + bioc->capacity = value; + bioc->data = g_realloc(bioc->data, bioc->capacity); + } + total_size = qemu_get_buffer(mis->from_src_file, bioc->data, value); + if (total_size != value) { + error_setg(errp, "Got %" PRIu64 " VMState data, less than expected" + " %" PRIu64, total_size, value); + return; + } + bioc->usage = total_size; + qio_channel_io_seek(QIO_CHANNEL(bioc), 0, 0, NULL); + + colo_send_message(mis->to_src_file, COLO_MESSAGE_VMSTATE_RECEIVED, + &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + + qemu_mutex_lock_iothread(); + vmstate_loading = true; + colo_flush_ram_cache(); + ret = qemu_load_device_state(fb); + if (ret < 0) { + error_setg(errp, "COLO: load device state failed"); + vmstate_loading = false; + qemu_mutex_unlock_iothread(); + return; + } + +#ifdef CONFIG_REPLICATION + replication_get_error_all(&local_err); + if (local_err) { + error_propagate(errp, local_err); + vmstate_loading = false; + qemu_mutex_unlock_iothread(); + return; + } + + /* discard colo disk buffer */ + replication_do_checkpoint_all(&local_err); + if (local_err) { + error_propagate(errp, local_err); + vmstate_loading = false; + qemu_mutex_unlock_iothread(); + return; + } +#else + abort(); +#endif + /* Notify all filters of all NIC to do checkpoint */ + colo_notify_filters_event(COLO_EVENT_CHECKPOINT, &local_err); + + if (local_err) { + error_propagate(errp, local_err); + vmstate_loading = false; + qemu_mutex_unlock_iothread(); + return; + } + + vmstate_loading = false; + vm_start(); + trace_colo_vm_state_change("stop", "run"); + qemu_mutex_unlock_iothread(); + + if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) { + return; + } + + colo_send_message(mis->to_src_file, COLO_MESSAGE_VMSTATE_LOADED, + &local_err); + error_propagate(errp, local_err); +} + +static void colo_wait_handle_message(MigrationIncomingState *mis, + QEMUFile *fb, QIOChannelBuffer *bioc, Error **errp) +{ + COLOMessage msg; + Error *local_err = NULL; + + msg = colo_receive_message(mis->from_src_file, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + + switch (msg) { + case COLO_MESSAGE_CHECKPOINT_REQUEST: + colo_incoming_process_checkpoint(mis, fb, bioc, errp); + break; + default: + error_setg(errp, "Got unknown COLO message: %d", msg); + break; + } +} + +void *colo_process_incoming_thread(void *opaque) +{ + MigrationIncomingState *mis = opaque; + QEMUFile *fb = NULL; + QIOChannelBuffer *bioc = NULL; /* Cache incoming device state */ + Error *local_err = NULL; + + rcu_register_thread(); + qemu_sem_init(&mis->colo_incoming_sem, 0); + + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_COLO); + + if (get_colo_mode() != COLO_MODE_SECONDARY) { + error_report("COLO mode must be COLO_MODE_SECONDARY"); + return NULL; + } + + failover_init_state(); + + mis->to_src_file = qemu_file_get_return_path(mis->from_src_file); + if (!mis->to_src_file) { + error_report("COLO incoming thread: Open QEMUFile to_src_file failed"); + goto out; + } + /* + * Note: the communication between Primary side and Secondary side + * should be sequential, we set the fd to unblocked in migration incoming + * coroutine, and here we are in the COLO incoming thread, so it is ok to + * set the fd back to blocked. + */ + qemu_file_set_blocking(mis->from_src_file, true); + + colo_incoming_start_dirty_log(); + + bioc = qio_channel_buffer_new(COLO_BUFFER_BASE_SIZE); + fb = qemu_fopen_channel_input(QIO_CHANNEL(bioc)); + object_unref(OBJECT(bioc)); + + qemu_mutex_lock_iothread(); +#ifdef CONFIG_REPLICATION + replication_start_all(REPLICATION_MODE_SECONDARY, &local_err); + if (local_err) { + qemu_mutex_unlock_iothread(); + goto out; + } +#else + abort(); +#endif + vm_start(); + trace_colo_vm_state_change("stop", "run"); + qemu_mutex_unlock_iothread(); + + colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_READY, + &local_err); + if (local_err) { + goto out; + } + + while (mis->state == MIGRATION_STATUS_COLO) { + colo_wait_handle_message(mis, fb, bioc, &local_err); + if (local_err) { + error_report_err(local_err); + break; + } + + if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) { + failover_set_state(FAILOVER_STATUS_RELAUNCH, + FAILOVER_STATUS_NONE); + failover_request_active(NULL); + break; + } + + if (failover_get_state() != FAILOVER_STATUS_NONE) { + error_report("failover request"); + break; + } + } + +out: + /* + * There are only two reasons we can get here, some error happened + * or the user triggered failover. + */ + switch (failover_get_state()) { + case FAILOVER_STATUS_COMPLETED: + qapi_event_send_colo_exit(COLO_MODE_SECONDARY, + COLO_EXIT_REASON_REQUEST); + break; + default: + qapi_event_send_colo_exit(COLO_MODE_SECONDARY, + COLO_EXIT_REASON_ERROR); + } + + if (fb) { + qemu_fclose(fb); + } + + /* Hope this not to be too long to loop here */ + qemu_sem_wait(&mis->colo_incoming_sem); + qemu_sem_destroy(&mis->colo_incoming_sem); + + rcu_unregister_thread(); + return NULL; +} diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c new file mode 100644 index 000000000..d65e744af --- /dev/null +++ b/migration/dirtyrate.c @@ -0,0 +1,795 @@ +/* + * Dirtyrate implement code + * + * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO.,LTD. + * + * Authors: + * Chuan Zheng <zhengchuan@huawei.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include <zlib.h> +#include "qapi/error.h" +#include "cpu.h" +#include "exec/ramblock.h" +#include "exec/ram_addr.h" +#include "qemu/rcu_queue.h" +#include "qemu/main-loop.h" +#include "qapi/qapi-commands-migration.h" +#include "ram.h" +#include "trace.h" +#include "dirtyrate.h" +#include "monitor/hmp.h" +#include "monitor/monitor.h" +#include "qapi/qmp/qdict.h" +#include "sysemu/kvm.h" +#include "sysemu/runstate.h" +#include "exec/memory.h" + +/* + * total_dirty_pages is procted by BQL and is used + * to stat dirty pages during the period of two + * memory_global_dirty_log_sync + */ +uint64_t total_dirty_pages; + +typedef struct DirtyPageRecord { + uint64_t start_pages; + uint64_t end_pages; +} DirtyPageRecord; + +static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; +static struct DirtyRateStat DirtyStat; +static DirtyRateMeasureMode dirtyrate_mode = + DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING; + +static int64_t set_sample_page_period(int64_t msec, int64_t initial_time) +{ + int64_t current_time; + + current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + if ((current_time - initial_time) >= msec) { + msec = current_time - initial_time; + } else { + g_usleep((msec + initial_time - current_time) * 1000); + } + + return msec; +} + +static bool is_sample_period_valid(int64_t sec) +{ + if (sec < MIN_FETCH_DIRTYRATE_TIME_SEC || + sec > MAX_FETCH_DIRTYRATE_TIME_SEC) { + return false; + } + + return true; +} + +static bool is_sample_pages_valid(int64_t pages) +{ + return pages >= MIN_SAMPLE_PAGE_COUNT && + pages <= MAX_SAMPLE_PAGE_COUNT; +} + +static int dirtyrate_set_state(int *state, int old_state, int new_state) +{ + assert(new_state < DIRTY_RATE_STATUS__MAX); + trace_dirtyrate_set_state(DirtyRateStatus_str(new_state)); + if (qatomic_cmpxchg(state, old_state, new_state) == old_state) { + return 0; + } else { + return -1; + } +} + +static struct DirtyRateInfo *query_dirty_rate_info(void) +{ + int i; + int64_t dirty_rate = DirtyStat.dirty_rate; + struct DirtyRateInfo *info = g_malloc0(sizeof(DirtyRateInfo)); + DirtyRateVcpuList *head = NULL, **tail = &head; + + info->status = CalculatingState; + info->start_time = DirtyStat.start_time; + info->calc_time = DirtyStat.calc_time; + info->sample_pages = DirtyStat.sample_pages; + info->mode = dirtyrate_mode; + + if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) { + info->has_dirty_rate = true; + info->dirty_rate = dirty_rate; + + if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { + /* + * set sample_pages with 0 to indicate page sampling + * isn't enabled + **/ + info->sample_pages = 0; + info->has_vcpu_dirty_rate = true; + for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) { + DirtyRateVcpu *rate = g_malloc0(sizeof(DirtyRateVcpu)); + rate->id = DirtyStat.dirty_ring.rates[i].id; + rate->dirty_rate = DirtyStat.dirty_ring.rates[i].dirty_rate; + QAPI_LIST_APPEND(tail, rate); + } + info->vcpu_dirty_rate = head; + } + + if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) { + info->sample_pages = 0; + } + } + + trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState)); + + return info; +} + +static void init_dirtyrate_stat(int64_t start_time, + struct DirtyRateConfig config) +{ + DirtyStat.dirty_rate = -1; + DirtyStat.start_time = start_time; + DirtyStat.calc_time = config.sample_period_seconds; + DirtyStat.sample_pages = config.sample_pages_per_gigabytes; + + switch (config.mode) { + case DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING: + DirtyStat.page_sampling.total_dirty_samples = 0; + DirtyStat.page_sampling.total_sample_count = 0; + DirtyStat.page_sampling.total_block_mem_MB = 0; + break; + case DIRTY_RATE_MEASURE_MODE_DIRTY_RING: + DirtyStat.dirty_ring.nvcpu = -1; + DirtyStat.dirty_ring.rates = NULL; + break; + default: + break; + } +} + +static void cleanup_dirtyrate_stat(struct DirtyRateConfig config) +{ + /* last calc-dirty-rate qmp use dirty ring mode */ + if (dirtyrate_mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { + free(DirtyStat.dirty_ring.rates); + DirtyStat.dirty_ring.rates = NULL; + } +} + +static void update_dirtyrate_stat(struct RamblockDirtyInfo *info) +{ + DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count; + DirtyStat.page_sampling.total_sample_count += info->sample_pages_count; + /* size of total pages in MB */ + DirtyStat.page_sampling.total_block_mem_MB += (info->ramblock_pages * + TARGET_PAGE_SIZE) >> 20; +} + +static void update_dirtyrate(uint64_t msec) +{ + uint64_t dirtyrate; + uint64_t total_dirty_samples = DirtyStat.page_sampling.total_dirty_samples; + uint64_t total_sample_count = DirtyStat.page_sampling.total_sample_count; + uint64_t total_block_mem_MB = DirtyStat.page_sampling.total_block_mem_MB; + + dirtyrate = total_dirty_samples * total_block_mem_MB * + 1000 / (total_sample_count * msec); + + DirtyStat.dirty_rate = dirtyrate; +} + +/* + * get hash result for the sampled memory with length of TARGET_PAGE_SIZE + * in ramblock, which starts from ramblock base address. + */ +static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, + uint64_t vfn) +{ + uint32_t crc; + + crc = crc32(0, (info->ramblock_addr + + vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE); + + trace_get_ramblock_vfn_hash(info->idstr, vfn, crc); + return crc; +} + +static bool save_ramblock_hash(struct RamblockDirtyInfo *info) +{ + unsigned int sample_pages_count; + int i; + GRand *rand; + + sample_pages_count = info->sample_pages_count; + + /* ramblock size less than one page, return success to skip this ramblock */ + if (unlikely(info->ramblock_pages == 0 || sample_pages_count == 0)) { + return true; + } + + info->hash_result = g_try_malloc0_n(sample_pages_count, + sizeof(uint32_t)); + if (!info->hash_result) { + return false; + } + + info->sample_page_vfn = g_try_malloc0_n(sample_pages_count, + sizeof(uint64_t)); + if (!info->sample_page_vfn) { + g_free(info->hash_result); + return false; + } + + rand = g_rand_new(); + for (i = 0; i < sample_pages_count; i++) { + info->sample_page_vfn[i] = g_rand_int_range(rand, 0, + info->ramblock_pages - 1); + info->hash_result[i] = get_ramblock_vfn_hash(info, + info->sample_page_vfn[i]); + } + g_rand_free(rand); + + return true; +} + +static void get_ramblock_dirty_info(RAMBlock *block, + struct RamblockDirtyInfo *info, + struct DirtyRateConfig *config) +{ + uint64_t sample_pages_per_gigabytes = config->sample_pages_per_gigabytes; + + /* Right shift 30 bits to calc ramblock size in GB */ + info->sample_pages_count = (qemu_ram_get_used_length(block) * + sample_pages_per_gigabytes) >> 30; + /* Right shift TARGET_PAGE_BITS to calc page count */ + info->ramblock_pages = qemu_ram_get_used_length(block) >> + TARGET_PAGE_BITS; + info->ramblock_addr = qemu_ram_get_host_addr(block); + strcpy(info->idstr, qemu_ram_get_idstr(block)); +} + +static void free_ramblock_dirty_info(struct RamblockDirtyInfo *infos, int count) +{ + int i; + + if (!infos) { + return; + } + + for (i = 0; i < count; i++) { + g_free(infos[i].sample_page_vfn); + g_free(infos[i].hash_result); + } + g_free(infos); +} + +static bool skip_sample_ramblock(RAMBlock *block) +{ + /* + * Sample only blocks larger than MIN_RAMBLOCK_SIZE. + */ + if (qemu_ram_get_used_length(block) < (MIN_RAMBLOCK_SIZE << 10)) { + trace_skip_sample_ramblock(block->idstr, + qemu_ram_get_used_length(block)); + return true; + } + + return false; +} + +static bool record_ramblock_hash_info(struct RamblockDirtyInfo **block_dinfo, + struct DirtyRateConfig config, + int *block_count) +{ + struct RamblockDirtyInfo *info = NULL; + struct RamblockDirtyInfo *dinfo = NULL; + RAMBlock *block = NULL; + int total_count = 0; + int index = 0; + bool ret = false; + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (skip_sample_ramblock(block)) { + continue; + } + total_count++; + } + + dinfo = g_try_malloc0_n(total_count, sizeof(struct RamblockDirtyInfo)); + if (dinfo == NULL) { + goto out; + } + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (skip_sample_ramblock(block)) { + continue; + } + if (index >= total_count) { + break; + } + info = &dinfo[index]; + get_ramblock_dirty_info(block, info, &config); + if (!save_ramblock_hash(info)) { + goto out; + } + index++; + } + ret = true; + +out: + *block_count = index; + *block_dinfo = dinfo; + return ret; +} + +static void calc_page_dirty_rate(struct RamblockDirtyInfo *info) +{ + uint32_t crc; + int i; + + for (i = 0; i < info->sample_pages_count; i++) { + crc = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); + if (crc != info->hash_result[i]) { + trace_calc_page_dirty_rate(info->idstr, crc, info->hash_result[i]); + info->sample_dirty_count++; + } + } +} + +static struct RamblockDirtyInfo * +find_block_matched(RAMBlock *block, int count, + struct RamblockDirtyInfo *infos) +{ + int i; + struct RamblockDirtyInfo *matched; + + for (i = 0; i < count; i++) { + if (!strcmp(infos[i].idstr, qemu_ram_get_idstr(block))) { + break; + } + } + + if (i == count) { + return NULL; + } + + if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) || + infos[i].ramblock_pages != + (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) { + trace_find_page_matched(block->idstr); + return NULL; + } + + matched = &infos[i]; + + return matched; +} + +static bool compare_page_hash_info(struct RamblockDirtyInfo *info, + int block_count) +{ + struct RamblockDirtyInfo *block_dinfo = NULL; + RAMBlock *block = NULL; + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + if (skip_sample_ramblock(block)) { + continue; + } + block_dinfo = find_block_matched(block, block_count, info); + if (block_dinfo == NULL) { + continue; + } + calc_page_dirty_rate(block_dinfo); + update_dirtyrate_stat(block_dinfo); + } + + if (DirtyStat.page_sampling.total_sample_count == 0) { + return false; + } + + return true; +} + +static inline void record_dirtypages(DirtyPageRecord *dirty_pages, + CPUState *cpu, bool start) +{ + if (start) { + dirty_pages[cpu->cpu_index].start_pages = cpu->dirty_pages; + } else { + dirty_pages[cpu->cpu_index].end_pages = cpu->dirty_pages; + } +} + +static void dirtyrate_global_dirty_log_start(void) +{ + qemu_mutex_lock_iothread(); + memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE); + qemu_mutex_unlock_iothread(); +} + +static void dirtyrate_global_dirty_log_stop(void) +{ + qemu_mutex_lock_iothread(); + memory_global_dirty_log_sync(); + memory_global_dirty_log_stop(GLOBAL_DIRTY_DIRTY_RATE); + qemu_mutex_unlock_iothread(); +} + +static int64_t do_calculate_dirtyrate_vcpu(DirtyPageRecord dirty_pages) +{ + uint64_t memory_size_MB; + int64_t time_s; + uint64_t increased_dirty_pages = + dirty_pages.end_pages - dirty_pages.start_pages; + + memory_size_MB = (increased_dirty_pages * TARGET_PAGE_SIZE) >> 20; + time_s = DirtyStat.calc_time; + + return memory_size_MB / time_s; +} + +static inline void record_dirtypages_bitmap(DirtyPageRecord *dirty_pages, + bool start) +{ + if (start) { + dirty_pages->start_pages = total_dirty_pages; + } else { + dirty_pages->end_pages = total_dirty_pages; + } +} + +static void do_calculate_dirtyrate_bitmap(DirtyPageRecord dirty_pages) +{ + DirtyStat.dirty_rate = do_calculate_dirtyrate_vcpu(dirty_pages); +} + +static inline void dirtyrate_manual_reset_protect(void) +{ + RAMBlock *block = NULL; + + WITH_RCU_READ_LOCK_GUARD() { + RAMBLOCK_FOREACH_MIGRATABLE(block) { + memory_region_clear_dirty_bitmap(block->mr, 0, + block->used_length); + } + } +} + +static void calculate_dirtyrate_dirty_bitmap(struct DirtyRateConfig config) +{ + int64_t msec = 0; + int64_t start_time; + DirtyPageRecord dirty_pages; + + qemu_mutex_lock_iothread(); + memory_global_dirty_log_start(GLOBAL_DIRTY_DIRTY_RATE); + + /* + * 1'round of log sync may return all 1 bits with + * KVM_DIRTY_LOG_INITIALLY_SET enable + * skip it unconditionally and start dirty tracking + * from 2'round of log sync + */ + memory_global_dirty_log_sync(); + + /* + * reset page protect manually and unconditionally. + * this make sure kvm dirty log be cleared if + * KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE cap is enabled. + */ + dirtyrate_manual_reset_protect(); + qemu_mutex_unlock_iothread(); + + record_dirtypages_bitmap(&dirty_pages, true); + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + DirtyStat.start_time = start_time / 1000; + + msec = config.sample_period_seconds * 1000; + msec = set_sample_page_period(msec, start_time); + DirtyStat.calc_time = msec / 1000; + + /* + * dirtyrate_global_dirty_log_stop do two things. + * 1. fetch dirty bitmap from kvm + * 2. stop dirty tracking + */ + dirtyrate_global_dirty_log_stop(); + + record_dirtypages_bitmap(&dirty_pages, false); + + do_calculate_dirtyrate_bitmap(dirty_pages); +} + +static void calculate_dirtyrate_dirty_ring(struct DirtyRateConfig config) +{ + CPUState *cpu; + int64_t msec = 0; + int64_t start_time; + uint64_t dirtyrate = 0; + uint64_t dirtyrate_sum = 0; + DirtyPageRecord *dirty_pages; + int nvcpu = 0; + int i = 0; + + CPU_FOREACH(cpu) { + nvcpu++; + } + + dirty_pages = malloc(sizeof(*dirty_pages) * nvcpu); + + DirtyStat.dirty_ring.nvcpu = nvcpu; + DirtyStat.dirty_ring.rates = malloc(sizeof(DirtyRateVcpu) * nvcpu); + + dirtyrate_global_dirty_log_start(); + + CPU_FOREACH(cpu) { + record_dirtypages(dirty_pages, cpu, true); + } + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + DirtyStat.start_time = start_time / 1000; + + msec = config.sample_period_seconds * 1000; + msec = set_sample_page_period(msec, start_time); + DirtyStat.calc_time = msec / 1000; + + dirtyrate_global_dirty_log_stop(); + + CPU_FOREACH(cpu) { + record_dirtypages(dirty_pages, cpu, false); + } + + for (i = 0; i < DirtyStat.dirty_ring.nvcpu; i++) { + dirtyrate = do_calculate_dirtyrate_vcpu(dirty_pages[i]); + trace_dirtyrate_do_calculate_vcpu(i, dirtyrate); + + DirtyStat.dirty_ring.rates[i].id = i; + DirtyStat.dirty_ring.rates[i].dirty_rate = dirtyrate; + dirtyrate_sum += dirtyrate; + } + + DirtyStat.dirty_rate = dirtyrate_sum; + free(dirty_pages); +} + +static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config) +{ + struct RamblockDirtyInfo *block_dinfo = NULL; + int block_count = 0; + int64_t msec = 0; + int64_t initial_time; + + rcu_read_lock(); + initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + if (!record_ramblock_hash_info(&block_dinfo, config, &block_count)) { + goto out; + } + rcu_read_unlock(); + + msec = config.sample_period_seconds * 1000; + msec = set_sample_page_period(msec, initial_time); + DirtyStat.start_time = initial_time / 1000; + DirtyStat.calc_time = msec / 1000; + + rcu_read_lock(); + if (!compare_page_hash_info(block_dinfo, block_count)) { + goto out; + } + + update_dirtyrate(msec); + +out: + rcu_read_unlock(); + free_ramblock_dirty_info(block_dinfo, block_count); +} + +static void calculate_dirtyrate(struct DirtyRateConfig config) +{ + if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) { + calculate_dirtyrate_dirty_bitmap(config); + } else if (config.mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { + calculate_dirtyrate_dirty_ring(config); + } else { + calculate_dirtyrate_sample_vm(config); + } + + trace_dirtyrate_calculate(DirtyStat.dirty_rate); +} + +void *get_dirtyrate_thread(void *arg) +{ + struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; + int ret; + rcu_register_thread(); + + ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_UNSTARTED, + DIRTY_RATE_STATUS_MEASURING); + if (ret == -1) { + error_report("change dirtyrate state failed."); + return NULL; + } + + calculate_dirtyrate(config); + + ret = dirtyrate_set_state(&CalculatingState, DIRTY_RATE_STATUS_MEASURING, + DIRTY_RATE_STATUS_MEASURED); + if (ret == -1) { + error_report("change dirtyrate state failed."); + } + + rcu_unregister_thread(); + return NULL; +} + +void qmp_calc_dirty_rate(int64_t calc_time, + bool has_sample_pages, + int64_t sample_pages, + bool has_mode, + DirtyRateMeasureMode mode, + Error **errp) +{ + static struct DirtyRateConfig config; + QemuThread thread; + int ret; + int64_t start_time; + + /* + * If the dirty rate is already being measured, don't attempt to start. + */ + if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURING) { + error_setg(errp, "the dirty rate is already being measured."); + return; + } + + if (!is_sample_period_valid(calc_time)) { + error_setg(errp, "calc-time is out of range[%d, %d].", + MIN_FETCH_DIRTYRATE_TIME_SEC, + MAX_FETCH_DIRTYRATE_TIME_SEC); + return; + } + + if (!has_mode) { + mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING; + } + + if (has_sample_pages && mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) { + error_setg(errp, "either sample-pages or dirty-ring can be specified."); + return; + } + + if (has_sample_pages) { + if (!is_sample_pages_valid(sample_pages)) { + error_setg(errp, "sample-pages is out of range[%d, %d].", + MIN_SAMPLE_PAGE_COUNT, + MAX_SAMPLE_PAGE_COUNT); + return; + } + } else { + sample_pages = DIRTYRATE_DEFAULT_SAMPLE_PAGES; + } + + /* + * dirty ring mode only works when kvm dirty ring is enabled. + * on the contrary, dirty bitmap mode is not. + */ + if (((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_RING) && + !kvm_dirty_ring_enabled()) || + ((mode == DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP) && + kvm_dirty_ring_enabled())) { + error_setg(errp, "mode %s is not enabled, use other method instead.", + DirtyRateMeasureMode_str(mode)); + return; + } + + /* + * Init calculation state as unstarted. + */ + ret = dirtyrate_set_state(&CalculatingState, CalculatingState, + DIRTY_RATE_STATUS_UNSTARTED); + if (ret == -1) { + error_setg(errp, "init dirty rate calculation state failed."); + return; + } + + config.sample_period_seconds = calc_time; + config.sample_pages_per_gigabytes = sample_pages; + config.mode = mode; + + cleanup_dirtyrate_stat(config); + + /* + * update dirty rate mode so that we can figure out what mode has + * been used in last calculation + **/ + dirtyrate_mode = mode; + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) / 1000; + init_dirtyrate_stat(start_time, config); + + qemu_thread_create(&thread, "get_dirtyrate", get_dirtyrate_thread, + (void *)&config, QEMU_THREAD_DETACHED); +} + +struct DirtyRateInfo *qmp_query_dirty_rate(Error **errp) +{ + return query_dirty_rate_info(); +} + +void hmp_info_dirty_rate(Monitor *mon, const QDict *qdict) +{ + DirtyRateInfo *info = query_dirty_rate_info(); + + monitor_printf(mon, "Status: %s\n", + DirtyRateStatus_str(info->status)); + monitor_printf(mon, "Start Time: %"PRIi64" (ms)\n", + info->start_time); + monitor_printf(mon, "Sample Pages: %"PRIu64" (per GB)\n", + info->sample_pages); + monitor_printf(mon, "Period: %"PRIi64" (sec)\n", + info->calc_time); + monitor_printf(mon, "Mode: %s\n", + DirtyRateMeasureMode_str(info->mode)); + monitor_printf(mon, "Dirty rate: "); + if (info->has_dirty_rate) { + monitor_printf(mon, "%"PRIi64" (MB/s)\n", info->dirty_rate); + if (info->has_vcpu_dirty_rate) { + DirtyRateVcpuList *rate, *head = info->vcpu_dirty_rate; + for (rate = head; rate != NULL; rate = rate->next) { + monitor_printf(mon, "vcpu[%"PRIi64"], Dirty rate: %"PRIi64 + " (MB/s)\n", rate->value->id, + rate->value->dirty_rate); + } + } + } else { + monitor_printf(mon, "(not ready)\n"); + } + + qapi_free_DirtyRateVcpuList(info->vcpu_dirty_rate); + g_free(info); +} + +void hmp_calc_dirty_rate(Monitor *mon, const QDict *qdict) +{ + int64_t sec = qdict_get_try_int(qdict, "second", 0); + int64_t sample_pages = qdict_get_try_int(qdict, "sample_pages_per_GB", -1); + bool has_sample_pages = (sample_pages != -1); + bool dirty_ring = qdict_get_try_bool(qdict, "dirty_ring", false); + bool dirty_bitmap = qdict_get_try_bool(qdict, "dirty_bitmap", false); + DirtyRateMeasureMode mode = DIRTY_RATE_MEASURE_MODE_PAGE_SAMPLING; + Error *err = NULL; + + if (!sec) { + monitor_printf(mon, "Incorrect period length specified!\n"); + return; + } + + if (dirty_ring && dirty_bitmap) { + monitor_printf(mon, "Either dirty ring or dirty bitmap " + "can be specified!\n"); + return; + } + + if (dirty_bitmap) { + mode = DIRTY_RATE_MEASURE_MODE_DIRTY_BITMAP; + } else if (dirty_ring) { + mode = DIRTY_RATE_MEASURE_MODE_DIRTY_RING; + } + + qmp_calc_dirty_rate(sec, has_sample_pages, sample_pages, true, + mode, &err); + if (err) { + hmp_handle_error(mon, err); + return; + } + + monitor_printf(mon, "Starting dirty rate measurement with period %"PRIi64 + " seconds\n", sec); + monitor_printf(mon, "[Please use 'info dirty_rate' to check results]\n"); +} diff --git a/migration/dirtyrate.h b/migration/dirtyrate.h new file mode 100644 index 000000000..69d4c5b86 --- /dev/null +++ b/migration/dirtyrate.h @@ -0,0 +1,88 @@ +/* + * Dirtyrate common functions + * + * Copyright (c) 2020 HUAWEI TECHNOLOGIES CO., LTD. + * + * Authors: + * Chuan Zheng <zhengchuan@huawei.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_MIGRATION_DIRTYRATE_H +#define QEMU_MIGRATION_DIRTYRATE_H + +/* + * Sample 512 pages per GB as default. + */ +#define DIRTYRATE_DEFAULT_SAMPLE_PAGES 512 + +/* + * Record ramblock idstr + */ +#define RAMBLOCK_INFO_MAX_LEN 256 + +/* + * Minimum RAMBlock size to sample, in megabytes. + */ +#define MIN_RAMBLOCK_SIZE 128 + +/* + * Take 1s as minimum time for calculation duration + */ +#define MIN_FETCH_DIRTYRATE_TIME_SEC 1 +#define MAX_FETCH_DIRTYRATE_TIME_SEC 60 + +/* + * Take 1/16 pages in 1G as the maxmum sample page count + */ +#define MIN_SAMPLE_PAGE_COUNT 128 +#define MAX_SAMPLE_PAGE_COUNT 16384 + +struct DirtyRateConfig { + uint64_t sample_pages_per_gigabytes; /* sample pages per GB */ + int64_t sample_period_seconds; /* time duration between two sampling */ + DirtyRateMeasureMode mode; /* mode of dirtyrate measurement */ +}; + +/* + * Store dirtypage info for each ramblock. + */ +struct RamblockDirtyInfo { + char idstr[RAMBLOCK_INFO_MAX_LEN]; /* idstr for each ramblock */ + uint8_t *ramblock_addr; /* base address of ramblock we measure */ + uint64_t ramblock_pages; /* ramblock size in TARGET_PAGE_SIZE */ + uint64_t *sample_page_vfn; /* relative offset address for sampled page */ + uint64_t sample_pages_count; /* count of sampled pages */ + uint64_t sample_dirty_count; /* count of dirty pages we measure */ + uint32_t *hash_result; /* array of hash result for sampled pages */ +}; + +typedef struct SampleVMStat { + uint64_t total_dirty_samples; /* total dirty sampled page */ + uint64_t total_sample_count; /* total sampled pages */ + uint64_t total_block_mem_MB; /* size of total sampled pages in MB */ +} SampleVMStat; + +typedef struct VcpuStat { + int nvcpu; /* number of vcpu */ + DirtyRateVcpu *rates; /* array of dirty rate for each vcpu */ +} VcpuStat; + +/* + * Store calculation statistics for each measure. + */ +struct DirtyRateStat { + int64_t dirty_rate; /* dirty rate in MB/s */ + int64_t start_time; /* calculation start time in units of second */ + int64_t calc_time; /* time duration of two sampling in units of second */ + uint64_t sample_pages; /* sample pages per GB */ + union { + SampleVMStat page_sampling; + VcpuStat dirty_ring; + }; +}; + +void *get_dirtyrate_thread(void *arg); +#endif diff --git a/migration/exec.c b/migration/exec.c new file mode 100644 index 000000000..375d2e1b5 --- /dev/null +++ b/migration/exec.c @@ -0,0 +1,73 @@ +/* + * QEMU live migration + * + * Copyright IBM, Corp. 2008 + * Copyright Dell MessageOne 2008 + * Copyright Red Hat, Inc. 2015-2016 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * Charles Duffy <charles_duffy@messageone.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "channel.h" +#include "exec.h" +#include "migration.h" +#include "io/channel-command.h" +#include "trace.h" + + +void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp) +{ + QIOChannel *ioc; + const char *argv[] = { "/bin/sh", "-c", command, NULL }; + + trace_migration_exec_outgoing(command); + ioc = QIO_CHANNEL(qio_channel_command_new_spawn(argv, + O_RDWR, + errp)); + if (!ioc) { + return; + } + + qio_channel_set_name(ioc, "migration-exec-outgoing"); + migration_channel_connect(s, ioc, NULL, NULL); + object_unref(OBJECT(ioc)); +} + +static gboolean exec_accept_incoming_migration(QIOChannel *ioc, + GIOCondition condition, + gpointer opaque) +{ + migration_channel_process_incoming(ioc); + object_unref(OBJECT(ioc)); + return G_SOURCE_REMOVE; +} + +void exec_start_incoming_migration(const char *command, Error **errp) +{ + QIOChannel *ioc; + const char *argv[] = { "/bin/sh", "-c", command, NULL }; + + trace_migration_exec_incoming(command); + ioc = QIO_CHANNEL(qio_channel_command_new_spawn(argv, + O_RDWR, + errp)); + if (!ioc) { + return; + } + + qio_channel_set_name(ioc, "migration-exec-incoming"); + qio_channel_add_watch_full(ioc, G_IO_IN, + exec_accept_incoming_migration, + NULL, NULL, + g_main_context_get_thread_default()); +} diff --git a/migration/exec.h b/migration/exec.h new file mode 100644 index 000000000..b210ffde7 --- /dev/null +++ b/migration/exec.h @@ -0,0 +1,26 @@ +/* + * QEMU live migration + * + * Copyright IBM, Corp. 2008 + * Copyright Dell MessageOne 2008 + * Copyright Red Hat, Inc. 2015-2016 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * Charles Duffy <charles_duffy@messageone.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#ifndef QEMU_MIGRATION_EXEC_H +#define QEMU_MIGRATION_EXEC_H +void exec_start_incoming_migration(const char *host_port, Error **errp); + +void exec_start_outgoing_migration(MigrationState *s, const char *host_port, + Error **errp); +#endif diff --git a/migration/fd.c b/migration/fd.c new file mode 100644 index 000000000..6f2f50475 --- /dev/null +++ b/migration/fd.c @@ -0,0 +1,76 @@ +/* + * QEMU live migration via generic fd + * + * Copyright Red Hat, Inc. 2009-2016 + * + * Authors: + * Chris Lalancette <clalance@redhat.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "channel.h" +#include "fd.h" +#include "migration.h" +#include "monitor/monitor.h" +#include "io/channel-util.h" +#include "trace.h" + + +void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp) +{ + QIOChannel *ioc; + int fd = monitor_get_fd(monitor_cur(), fdname, errp); + if (fd == -1) { + return; + } + + trace_migration_fd_outgoing(fd); + ioc = qio_channel_new_fd(fd, errp); + if (!ioc) { + close(fd); + return; + } + + qio_channel_set_name(QIO_CHANNEL(ioc), "migration-fd-outgoing"); + migration_channel_connect(s, ioc, NULL, NULL); + object_unref(OBJECT(ioc)); +} + +static gboolean fd_accept_incoming_migration(QIOChannel *ioc, + GIOCondition condition, + gpointer opaque) +{ + migration_channel_process_incoming(ioc); + object_unref(OBJECT(ioc)); + return G_SOURCE_REMOVE; +} + +void fd_start_incoming_migration(const char *fdname, Error **errp) +{ + QIOChannel *ioc; + int fd = monitor_fd_param(monitor_cur(), fdname, errp); + if (fd == -1) { + return; + } + + trace_migration_fd_incoming(fd); + + ioc = qio_channel_new_fd(fd, errp); + if (!ioc) { + close(fd); + return; + } + + qio_channel_set_name(QIO_CHANNEL(ioc), "migration-fd-incoming"); + qio_channel_add_watch_full(ioc, G_IO_IN, + fd_accept_incoming_migration, + NULL, NULL, + g_main_context_get_thread_default()); +} diff --git a/migration/fd.h b/migration/fd.h new file mode 100644 index 000000000..b901bc014 --- /dev/null +++ b/migration/fd.h @@ -0,0 +1,23 @@ +/* + * QEMU live migration via generic fd + * + * Copyright Red Hat, Inc. 2009-2016 + * + * Authors: + * Chris Lalancette <clalance@redhat.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#ifndef QEMU_MIGRATION_FD_H +#define QEMU_MIGRATION_FD_H +void fd_start_incoming_migration(const char *fdname, Error **errp); + +void fd_start_outgoing_migration(MigrationState *s, const char *fdname, + Error **errp); +#endif diff --git a/migration/global_state.c b/migration/global_state.c new file mode 100644 index 000000000..a33947ca3 --- /dev/null +++ b/migration/global_state.c @@ -0,0 +1,148 @@ +/* + * Global State configuration + * + * Copyright (c) 2014-2017 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu/cutils.h" +#include "qemu/error-report.h" +#include "sysemu/runstate.h" +#include "qapi/error.h" +#include "migration.h" +#include "migration/global_state.h" +#include "migration/vmstate.h" +#include "trace.h" + +typedef struct { + uint32_t size; + uint8_t runstate[100]; + RunState state; + bool received; +} GlobalState; + +static GlobalState global_state; + +int global_state_store(void) +{ + if (!runstate_store((char *)global_state.runstate, + sizeof(global_state.runstate))) { + error_report("runstate name too big: %s", global_state.runstate); + trace_migrate_state_too_big(); + return -EINVAL; + } + return 0; +} + +void global_state_store_running(void) +{ + const char *state = RunState_str(RUN_STATE_RUNNING); + assert(strlen(state) < sizeof(global_state.runstate)); + strpadcpy((char *)global_state.runstate, sizeof(global_state.runstate), + state, '\0'); +} + +bool global_state_received(void) +{ + return global_state.received; +} + +RunState global_state_get_runstate(void) +{ + return global_state.state; +} + +static bool global_state_needed(void *opaque) +{ + GlobalState *s = opaque; + char *runstate = (char *)s->runstate; + + /* If it is not optional, it is mandatory */ + + if (migrate_get_current()->store_global_state) { + return true; + } + + /* If state is running or paused, it is not needed */ + + if (strcmp(runstate, "running") == 0 || + strcmp(runstate, "paused") == 0) { + return false; + } + + /* for any other state it is needed */ + return true; +} + +static int global_state_post_load(void *opaque, int version_id) +{ + GlobalState *s = opaque; + Error *local_err = NULL; + int r; + char *runstate = (char *)s->runstate; + + s->received = true; + trace_migrate_global_state_post_load(runstate); + + if (strnlen((char *)s->runstate, + sizeof(s->runstate)) == sizeof(s->runstate)) { + /* + * This condition should never happen during migration, because + * all runstate names are shorter than 100 bytes (the size of + * s->runstate). However, a malicious stream could overflow + * the qapi_enum_parse() call, so we force the last character + * to a NUL byte. + */ + s->runstate[sizeof(s->runstate) - 1] = '\0'; + } + r = qapi_enum_parse(&RunState_lookup, runstate, -1, &local_err); + + if (r == -1) { + if (local_err) { + error_report_err(local_err); + } + return -EINVAL; + } + s->state = r; + + return 0; +} + +static int global_state_pre_save(void *opaque) +{ + GlobalState *s = opaque; + + trace_migrate_global_state_pre_save((char *)s->runstate); + s->size = strnlen((char *)s->runstate, sizeof(s->runstate)) + 1; + assert(s->size <= sizeof(s->runstate)); + + return 0; +} + +static const VMStateDescription vmstate_globalstate = { + .name = "globalstate", + .version_id = 1, + .minimum_version_id = 1, + .post_load = global_state_post_load, + .pre_save = global_state_pre_save, + .needed = global_state_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT32(size, GlobalState), + VMSTATE_BUFFER(runstate, GlobalState), + VMSTATE_END_OF_LIST() + }, +}; + +void register_global_state(void) +{ + /* We would use it independently that we receive it */ + strcpy((char *)&global_state.runstate, ""); + global_state.received = false; + vmstate_register(NULL, 0, &vmstate_globalstate, &global_state); +} diff --git a/migration/meson.build b/migration/meson.build new file mode 100644 index 000000000..f8714dcb1 --- /dev/null +++ b/migration/meson.build @@ -0,0 +1,35 @@ +# Files needed by unit tests +migration_files = files( + 'page_cache.c', + 'xbzrle.c', + 'vmstate-types.c', + 'vmstate.c', + 'qemu-file-channel.c', + 'qemu-file.c', + 'yank_functions.c', +) +softmmu_ss.add(migration_files) + +softmmu_ss.add(files( + 'block-dirty-bitmap.c', + 'channel.c', + 'colo-failover.c', + 'colo.c', + 'exec.c', + 'fd.c', + 'global_state.c', + 'migration.c', + 'multifd.c', + 'multifd-zlib.c', + 'postcopy-ram.c', + 'savevm.c', + 'socket.c', + 'tls.c', +), gnutls) + +softmmu_ss.add(when: ['CONFIG_RDMA', rdma], if_true: files('rdma.c')) +softmmu_ss.add(when: 'CONFIG_LIVE_BLOCK_MIGRATION', if_true: files('block.c')) +softmmu_ss.add(when: zstd, if_true: files('multifd-zstd.c')) + +specific_ss.add(when: 'CONFIG_SOFTMMU', + if_true: files('dirtyrate.c', 'ram.c', 'target.c')) diff --git a/migration/migration.c b/migration/migration.c new file mode 100644 index 000000000..abaf6f9e3 --- /dev/null +++ b/migration/migration.c @@ -0,0 +1,4358 @@ +/* + * QEMU live migration + * + * Copyright IBM, Corp. 2008 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "qemu/cutils.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" +#include "migration/blocker.h" +#include "exec.h" +#include "fd.h" +#include "socket.h" +#include "sysemu/runstate.h" +#include "sysemu/sysemu.h" +#include "sysemu/cpu-throttle.h" +#include "rdma.h" +#include "ram.h" +#include "migration/global_state.h" +#include "migration/misc.h" +#include "migration.h" +#include "savevm.h" +#include "qemu-file-channel.h" +#include "qemu-file.h" +#include "migration/vmstate.h" +#include "block/block.h" +#include "qapi/error.h" +#include "qapi/clone-visitor.h" +#include "qapi/qapi-visit-migration.h" +#include "qapi/qapi-visit-sockets.h" +#include "qapi/qapi-commands-migration.h" +#include "qapi/qapi-events-migration.h" +#include "qapi/qmp/qerror.h" +#include "qapi/qmp/qnull.h" +#include "qemu/rcu.h" +#include "block.h" +#include "postcopy-ram.h" +#include "qemu/thread.h" +#include "trace.h" +#include "exec/target_page.h" +#include "io/channel-buffer.h" +#include "migration/colo.h" +#include "hw/boards.h" +#include "hw/qdev-properties.h" +#include "hw/qdev-properties-system.h" +#include "monitor/monitor.h" +#include "net/announce.h" +#include "qemu/queue.h" +#include "multifd.h" +#include "qemu/yank.h" +#include "sysemu/cpus.h" +#include "yank_functions.h" + +#define MAX_THROTTLE (128 << 20) /* Migration transfer speed throttling */ + +/* Amount of time to allocate to each "chunk" of bandwidth-throttled + * data. */ +#define BUFFER_DELAY 100 +#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY) + +/* Time in milliseconds we are allowed to stop the source, + * for sending the last part */ +#define DEFAULT_MIGRATE_SET_DOWNTIME 300 + +/* Maximum migrate downtime set to 2000 seconds */ +#define MAX_MIGRATE_DOWNTIME_SECONDS 2000 +#define MAX_MIGRATE_DOWNTIME (MAX_MIGRATE_DOWNTIME_SECONDS * 1000) + +/* Default compression thread count */ +#define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8 +/* Default decompression thread count, usually decompression is at + * least 4 times as fast as compression.*/ +#define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2 +/*0: means nocompress, 1: best speed, ... 9: best compress ratio */ +#define DEFAULT_MIGRATE_COMPRESS_LEVEL 1 +/* Define default autoconverge cpu throttle migration parameters */ +#define DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD 50 +#define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20 +#define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10 +#define DEFAULT_MIGRATE_MAX_CPU_THROTTLE 99 + +/* Migration XBZRLE default cache size */ +#define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024) + +/* The delay time (in ms) between two COLO checkpoints */ +#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100) +#define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2 +#define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE +/* 0: means nocompress, 1: best speed, ... 9: best compress ratio */ +#define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1 +/* 0: means nocompress, 1: best speed, ... 20: best compress ratio */ +#define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1 + +/* Background transfer rate for postcopy, 0 means unlimited, note + * that page requests can still exceed this limit. + */ +#define DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH 0 + +/* + * Parameters for self_announce_delay giving a stream of RARP/ARP + * packets after migration. + */ +#define DEFAULT_MIGRATE_ANNOUNCE_INITIAL 50 +#define DEFAULT_MIGRATE_ANNOUNCE_MAX 550 +#define DEFAULT_MIGRATE_ANNOUNCE_ROUNDS 5 +#define DEFAULT_MIGRATE_ANNOUNCE_STEP 100 + +static NotifierList migration_state_notifiers = + NOTIFIER_LIST_INITIALIZER(migration_state_notifiers); + +/* Messages sent on the return path from destination to source */ +enum mig_rp_message_type { + MIG_RP_MSG_INVALID = 0, /* Must be 0 */ + MIG_RP_MSG_SHUT, /* sibling will not send any more RP messages */ + MIG_RP_MSG_PONG, /* Response to a PING; data (seq: be32 ) */ + + MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */ + MIG_RP_MSG_REQ_PAGES, /* data (start: be64, len: be32) */ + MIG_RP_MSG_RECV_BITMAP, /* send recved_bitmap back to source */ + MIG_RP_MSG_RESUME_ACK, /* tell source that we are ready to resume */ + + MIG_RP_MSG_MAX +}; + +/* Migration capabilities set */ +struct MigrateCapsSet { + int size; /* Capability set size */ + MigrationCapability caps[]; /* Variadic array of capabilities */ +}; +typedef struct MigrateCapsSet MigrateCapsSet; + +/* Define and initialize MigrateCapsSet */ +#define INITIALIZE_MIGRATE_CAPS_SET(_name, ...) \ + MigrateCapsSet _name = { \ + .size = sizeof((int []) { __VA_ARGS__ }) / sizeof(int), \ + .caps = { __VA_ARGS__ } \ + } + +/* Background-snapshot compatibility check list */ +static const +INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot, + MIGRATION_CAPABILITY_POSTCOPY_RAM, + MIGRATION_CAPABILITY_DIRTY_BITMAPS, + MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME, + MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE, + MIGRATION_CAPABILITY_RETURN_PATH, + MIGRATION_CAPABILITY_MULTIFD, + MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER, + MIGRATION_CAPABILITY_AUTO_CONVERGE, + MIGRATION_CAPABILITY_RELEASE_RAM, + MIGRATION_CAPABILITY_RDMA_PIN_ALL, + MIGRATION_CAPABILITY_COMPRESS, + MIGRATION_CAPABILITY_XBZRLE, + MIGRATION_CAPABILITY_X_COLO, + MIGRATION_CAPABILITY_VALIDATE_UUID); + +/* When we add fault tolerance, we could have several + migrations at once. For now we don't need to add + dynamic creation of migration */ + +static MigrationState *current_migration; +static MigrationIncomingState *current_incoming; + +static GSList *migration_blockers; + +static bool migration_object_check(MigrationState *ms, Error **errp); +static int migration_maybe_pause(MigrationState *s, + int *current_active_state, + int new_state); +static void migrate_fd_cancel(MigrationState *s); + +static gint page_request_addr_cmp(gconstpointer ap, gconstpointer bp) +{ + uintptr_t a = (uintptr_t) ap, b = (uintptr_t) bp; + + return (a > b) - (a < b); +} + +void migration_object_init(void) +{ + /* This can only be called once. */ + assert(!current_migration); + current_migration = MIGRATION_OBJ(object_new(TYPE_MIGRATION)); + + /* + * Init the migrate incoming object as well no matter whether + * we'll use it or not. + */ + assert(!current_incoming); + current_incoming = g_new0(MigrationIncomingState, 1); + current_incoming->state = MIGRATION_STATUS_NONE; + current_incoming->postcopy_remote_fds = + g_array_new(FALSE, TRUE, sizeof(struct PostCopyFD)); + qemu_mutex_init(¤t_incoming->rp_mutex); + qemu_event_init(¤t_incoming->main_thread_load_event, false); + qemu_sem_init(¤t_incoming->postcopy_pause_sem_dst, 0); + qemu_sem_init(¤t_incoming->postcopy_pause_sem_fault, 0); + qemu_mutex_init(¤t_incoming->page_request_mutex); + current_incoming->page_requested = g_tree_new(page_request_addr_cmp); + + migration_object_check(current_migration, &error_fatal); + + blk_mig_init(); + ram_mig_init(); + dirty_bitmap_mig_init(); +} + +void migration_cancel(const Error *error) +{ + if (error) { + migrate_set_error(current_migration, error); + } + migrate_fd_cancel(current_migration); +} + +void migration_shutdown(void) +{ + /* + * Cancel the current migration - that will (eventually) + * stop the migration using this structure + */ + migration_cancel(NULL); + object_unref(OBJECT(current_migration)); + + /* + * Cancel outgoing migration of dirty bitmaps. It should + * at least unref used block nodes. + */ + dirty_bitmap_mig_cancel_outgoing(); + + /* + * Cancel incoming migration of dirty bitmaps. Dirty bitmaps + * are non-critical data, and their loss never considered as + * something serious. + */ + dirty_bitmap_mig_cancel_incoming(); +} + +/* For outgoing */ +MigrationState *migrate_get_current(void) +{ + /* This can only be called after the object created. */ + assert(current_migration); + return current_migration; +} + +MigrationIncomingState *migration_incoming_get_current(void) +{ + assert(current_incoming); + return current_incoming; +} + +void migration_incoming_state_destroy(void) +{ + struct MigrationIncomingState *mis = migration_incoming_get_current(); + + if (mis->to_src_file) { + /* Tell source that we are done */ + migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0); + qemu_fclose(mis->to_src_file); + mis->to_src_file = NULL; + } + + if (mis->from_src_file) { + migration_ioc_unregister_yank_from_file(mis->from_src_file); + qemu_fclose(mis->from_src_file); + mis->from_src_file = NULL; + } + if (mis->postcopy_remote_fds) { + g_array_free(mis->postcopy_remote_fds, TRUE); + mis->postcopy_remote_fds = NULL; + } + if (mis->transport_cleanup) { + mis->transport_cleanup(mis->transport_data); + } + + qemu_event_reset(&mis->main_thread_load_event); + + if (mis->page_requested) { + g_tree_destroy(mis->page_requested); + mis->page_requested = NULL; + } + + if (mis->socket_address_list) { + qapi_free_SocketAddressList(mis->socket_address_list); + mis->socket_address_list = NULL; + } + + yank_unregister_instance(MIGRATION_YANK_INSTANCE); +} + +static void migrate_generate_event(int new_state) +{ + if (migrate_use_events()) { + qapi_event_send_migration(new_state); + } +} + +static bool migrate_late_block_activate(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[ + MIGRATION_CAPABILITY_LATE_BLOCK_ACTIVATE]; +} + +/* + * Send a message on the return channel back to the source + * of the migration. + */ +static int migrate_send_rp_message(MigrationIncomingState *mis, + enum mig_rp_message_type message_type, + uint16_t len, void *data) +{ + int ret = 0; + + trace_migrate_send_rp_message((int)message_type, len); + QEMU_LOCK_GUARD(&mis->rp_mutex); + + /* + * It's possible that the file handle got lost due to network + * failures. + */ + if (!mis->to_src_file) { + ret = -EIO; + return ret; + } + + qemu_put_be16(mis->to_src_file, (unsigned int)message_type); + qemu_put_be16(mis->to_src_file, len); + qemu_put_buffer(mis->to_src_file, data, len); + qemu_fflush(mis->to_src_file); + + /* It's possible that qemu file got error during sending */ + ret = qemu_file_get_error(mis->to_src_file); + + return ret; +} + +/* Request one page from the source VM at the given start address. + * rb: the RAMBlock to request the page in + * Start: Address offset within the RB + * Len: Length in bytes required - must be a multiple of pagesize + */ +int migrate_send_rp_message_req_pages(MigrationIncomingState *mis, + RAMBlock *rb, ram_addr_t start) +{ + uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */ + size_t msglen = 12; /* start + len */ + size_t len = qemu_ram_pagesize(rb); + enum mig_rp_message_type msg_type; + const char *rbname; + int rbname_len; + + *(uint64_t *)bufc = cpu_to_be64((uint64_t)start); + *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len); + + /* + * We maintain the last ramblock that we requested for page. Note that we + * don't need locking because this function will only be called within the + * postcopy ram fault thread. + */ + if (rb != mis->last_rb) { + mis->last_rb = rb; + + rbname = qemu_ram_get_idstr(rb); + rbname_len = strlen(rbname); + + assert(rbname_len < 256); + + bufc[msglen++] = rbname_len; + memcpy(bufc + msglen, rbname, rbname_len); + msglen += rbname_len; + msg_type = MIG_RP_MSG_REQ_PAGES_ID; + } else { + msg_type = MIG_RP_MSG_REQ_PAGES; + } + + return migrate_send_rp_message(mis, msg_type, msglen, bufc); +} + +int migrate_send_rp_req_pages(MigrationIncomingState *mis, + RAMBlock *rb, ram_addr_t start, uint64_t haddr) +{ + void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb)); + bool received = false; + + WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) { + received = ramblock_recv_bitmap_test_byte_offset(rb, start); + if (!received && !g_tree_lookup(mis->page_requested, aligned)) { + /* + * The page has not been received, and it's not yet in the page + * request list. Queue it. Set the value of element to 1, so that + * things like g_tree_lookup() will return TRUE (1) when found. + */ + g_tree_insert(mis->page_requested, aligned, (gpointer)1); + mis->page_requested_count++; + trace_postcopy_page_req_add(aligned, mis->page_requested_count); + } + } + + /* + * If the page is there, skip sending the message. We don't even need the + * lock because as long as the page arrived, it'll be there forever. + */ + if (received) { + return 0; + } + + return migrate_send_rp_message_req_pages(mis, rb, start); +} + +static bool migration_colo_enabled; +bool migration_incoming_colo_enabled(void) +{ + return migration_colo_enabled; +} + +void migration_incoming_disable_colo(void) +{ + ram_block_discard_disable(false); + migration_colo_enabled = false; +} + +int migration_incoming_enable_colo(void) +{ + if (ram_block_discard_disable(true)) { + error_report("COLO: cannot disable RAM discard"); + return -EBUSY; + } + migration_colo_enabled = true; + return 0; +} + +void migrate_add_address(SocketAddress *address) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + QAPI_LIST_PREPEND(mis->socket_address_list, + QAPI_CLONE(SocketAddress, address)); +} + +static void qemu_start_incoming_migration(const char *uri, Error **errp) +{ + const char *p = NULL; + + migrate_protocol_allow_multifd(false); /* reset it anyway */ + qapi_event_send_migration(MIGRATION_STATUS_SETUP); + if (strstart(uri, "tcp:", &p) || + strstart(uri, "unix:", NULL) || + strstart(uri, "vsock:", NULL)) { + migrate_protocol_allow_multifd(true); + socket_start_incoming_migration(p ? p : uri, errp); +#ifdef CONFIG_RDMA + } else if (strstart(uri, "rdma:", &p)) { + rdma_start_incoming_migration(p, errp); +#endif + } else if (strstart(uri, "exec:", &p)) { + exec_start_incoming_migration(p, errp); + } else if (strstart(uri, "fd:", &p)) { + fd_start_incoming_migration(p, errp); + } else { + error_setg(errp, "unknown migration protocol: %s", uri); + } +} + +static void process_incoming_migration_bh(void *opaque) +{ + Error *local_err = NULL; + MigrationIncomingState *mis = opaque; + + /* If capability late_block_activate is set: + * Only fire up the block code now if we're going to restart the + * VM, else 'cont' will do it. + * This causes file locking to happen; so we don't want it to happen + * unless we really are starting the VM. + */ + if (!migrate_late_block_activate() || + (autostart && (!global_state_received() || + global_state_get_runstate() == RUN_STATE_RUNNING))) { + /* Make sure all file formats flush their mutable metadata. + * If we get an error here, just don't restart the VM yet. */ + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + local_err = NULL; + autostart = false; + } + } + + /* + * This must happen after all error conditions are dealt with and + * we're sure the VM is going to be running on this host. + */ + qemu_announce_self(&mis->announce_timer, migrate_announce_params()); + + if (multifd_load_cleanup(&local_err) != 0) { + error_report_err(local_err); + autostart = false; + } + /* If global state section was not received or we are in running + state, we need to obey autostart. Any other state is set with + runstate_set. */ + + dirty_bitmap_mig_before_vm_start(); + + if (!global_state_received() || + global_state_get_runstate() == RUN_STATE_RUNNING) { + if (autostart) { + vm_start(); + } else { + runstate_set(RUN_STATE_PAUSED); + } + } else if (migration_incoming_colo_enabled()) { + migration_incoming_disable_colo(); + vm_start(); + } else { + runstate_set(global_state_get_runstate()); + } + /* + * This must happen after any state changes since as soon as an external + * observer sees this event they might start to prod at the VM assuming + * it's ready to use. + */ + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_COMPLETED); + qemu_bh_delete(mis->bh); + migration_incoming_state_destroy(); +} + +static void process_incoming_migration_co(void *opaque) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + PostcopyState ps; + int ret; + Error *local_err = NULL; + + assert(mis->from_src_file); + mis->migration_incoming_co = qemu_coroutine_self(); + mis->largest_page_size = qemu_ram_pagesize_largest(); + postcopy_state_set(POSTCOPY_INCOMING_NONE); + migrate_set_state(&mis->state, MIGRATION_STATUS_NONE, + MIGRATION_STATUS_ACTIVE); + ret = qemu_loadvm_state(mis->from_src_file); + + ps = postcopy_state_get(); + trace_process_incoming_migration_co_end(ret, ps); + if (ps != POSTCOPY_INCOMING_NONE) { + if (ps == POSTCOPY_INCOMING_ADVISE) { + /* + * Where a migration had postcopy enabled (and thus went to advise) + * but managed to complete within the precopy period, we can use + * the normal exit. + */ + postcopy_ram_incoming_cleanup(mis); + } else if (ret >= 0) { + /* + * Postcopy was started, cleanup should happen at the end of the + * postcopy thread. + */ + trace_process_incoming_migration_co_postcopy_end_main(); + return; + } + /* Else if something went wrong then just fall out of the normal exit */ + } + + /* we get COLO info, and know if we are in COLO mode */ + if (!ret && migration_incoming_colo_enabled()) { + /* Make sure all file formats flush their mutable metadata */ + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + goto fail; + } + + qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming", + colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE); + mis->have_colo_incoming_thread = true; + qemu_coroutine_yield(); + + qemu_mutex_unlock_iothread(); + /* Wait checkpoint incoming thread exit before free resource */ + qemu_thread_join(&mis->colo_incoming_thread); + qemu_mutex_lock_iothread(); + /* We hold the global iothread lock, so it is safe here */ + colo_release_ram_cache(); + } + + if (ret < 0) { + error_report("load of migration failed: %s", strerror(-ret)); + goto fail; + } + mis->bh = qemu_bh_new(process_incoming_migration_bh, mis); + qemu_bh_schedule(mis->bh); + mis->migration_incoming_co = NULL; + return; +fail: + local_err = NULL; + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_FAILED); + qemu_fclose(mis->from_src_file); + if (multifd_load_cleanup(&local_err) != 0) { + error_report_err(local_err); + } + exit(EXIT_FAILURE); +} + +/** + * migration_incoming_setup: Setup incoming migration + * @f: file for main migration channel + * @errp: where to put errors + * + * Returns: %true on success, %false on error. + */ +static bool migration_incoming_setup(QEMUFile *f, Error **errp) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + if (multifd_load_setup(errp) != 0) { + return false; + } + + if (!mis->from_src_file) { + mis->from_src_file = f; + } + qemu_file_set_blocking(f, false); + return true; +} + +void migration_incoming_process(void) +{ + Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, NULL); + qemu_coroutine_enter(co); +} + +/* Returns true if recovered from a paused migration, otherwise false */ +static bool postcopy_try_recover(QEMUFile *f) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + if (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) { + /* Resumed from a paused postcopy migration */ + + mis->from_src_file = f; + /* Postcopy has standalone thread to do vm load */ + qemu_file_set_blocking(f, true); + + /* Re-configure the return path */ + mis->to_src_file = qemu_file_get_return_path(f); + + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_PAUSED, + MIGRATION_STATUS_POSTCOPY_RECOVER); + + /* + * Here, we only wake up the main loading thread (while the + * fault thread will still be waiting), so that we can receive + * commands from source now, and answer it if needed. The + * fault thread will be woken up afterwards until we are sure + * that source is ready to reply to page requests. + */ + qemu_sem_post(&mis->postcopy_pause_sem_dst); + return true; + } + + return false; +} + +void migration_fd_process_incoming(QEMUFile *f, Error **errp) +{ + if (postcopy_try_recover(f)) { + return; + } + + if (!migration_incoming_setup(f, errp)) { + return; + } + migration_incoming_process(); +} + +void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + Error *local_err = NULL; + bool start_migration; + + if (!mis->from_src_file) { + /* The first connection (multifd may have multiple) */ + QEMUFile *f = qemu_fopen_channel_input(ioc); + + /* If it's a recovery, we're done */ + if (postcopy_try_recover(f)) { + return; + } + + if (!migration_incoming_setup(f, errp)) { + return; + } + + /* + * Common migration only needs one channel, so we can start + * right now. Multifd needs more than one channel, we wait. + */ + start_migration = !migrate_use_multifd(); + } else { + /* Multiple connections */ + assert(migrate_use_multifd()); + start_migration = multifd_recv_new_channel(ioc, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return; + } + } + + if (start_migration) { + migration_incoming_process(); + } +} + +/** + * @migration_has_all_channels: We have received all channels that we need + * + * Returns true when we have got connections to all the channels that + * we need for migration. + */ +bool migration_has_all_channels(void) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + bool all_channels; + + all_channels = multifd_recv_all_channels_created(); + + return all_channels && mis->from_src_file != NULL; +} + +/* + * Send a 'SHUT' message on the return channel with the given value + * to indicate that we've finished with the RP. Non-0 value indicates + * error. + */ +void migrate_send_rp_shut(MigrationIncomingState *mis, + uint32_t value) +{ + uint32_t buf; + + buf = cpu_to_be32(value); + migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf); +} + +/* + * Send a 'PONG' message on the return channel with the given value + * (normally in response to a 'PING') + */ +void migrate_send_rp_pong(MigrationIncomingState *mis, + uint32_t value) +{ + uint32_t buf; + + buf = cpu_to_be32(value); + migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf); +} + +void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis, + char *block_name) +{ + char buf[512]; + int len; + int64_t res; + + /* + * First, we send the header part. It contains only the len of + * idstr, and the idstr itself. + */ + len = strlen(block_name); + buf[0] = len; + memcpy(buf + 1, block_name, len); + + if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) { + error_report("%s: MSG_RP_RECV_BITMAP only used for recovery", + __func__); + return; + } + + migrate_send_rp_message(mis, MIG_RP_MSG_RECV_BITMAP, len + 1, buf); + + /* + * Next, we dump the received bitmap to the stream. + * + * TODO: currently we are safe since we are the only one that is + * using the to_src_file handle (fault thread is still paused), + * and it's ok even not taking the mutex. However the best way is + * to take the lock before sending the message header, and release + * the lock after sending the bitmap. + */ + qemu_mutex_lock(&mis->rp_mutex); + res = ramblock_recv_bitmap_send(mis->to_src_file, block_name); + qemu_mutex_unlock(&mis->rp_mutex); + + trace_migrate_send_rp_recv_bitmap(block_name, res); +} + +void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value) +{ + uint32_t buf; + + buf = cpu_to_be32(value); + migrate_send_rp_message(mis, MIG_RP_MSG_RESUME_ACK, sizeof(buf), &buf); +} + +MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp) +{ + MigrationCapabilityStatusList *head = NULL, **tail = &head; + MigrationCapabilityStatus *caps; + MigrationState *s = migrate_get_current(); + int i; + + for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { +#ifndef CONFIG_LIVE_BLOCK_MIGRATION + if (i == MIGRATION_CAPABILITY_BLOCK) { + continue; + } +#endif + caps = g_malloc0(sizeof(*caps)); + caps->capability = i; + caps->state = s->enabled_capabilities[i]; + QAPI_LIST_APPEND(tail, caps); + } + + return head; +} + +MigrationParameters *qmp_query_migrate_parameters(Error **errp) +{ + MigrationParameters *params; + MigrationState *s = migrate_get_current(); + + /* TODO use QAPI_CLONE() instead of duplicating it inline */ + params = g_malloc0(sizeof(*params)); + params->has_compress_level = true; + params->compress_level = s->parameters.compress_level; + params->has_compress_threads = true; + params->compress_threads = s->parameters.compress_threads; + params->has_compress_wait_thread = true; + params->compress_wait_thread = s->parameters.compress_wait_thread; + params->has_decompress_threads = true; + params->decompress_threads = s->parameters.decompress_threads; + params->has_throttle_trigger_threshold = true; + params->throttle_trigger_threshold = s->parameters.throttle_trigger_threshold; + params->has_cpu_throttle_initial = true; + params->cpu_throttle_initial = s->parameters.cpu_throttle_initial; + params->has_cpu_throttle_increment = true; + params->cpu_throttle_increment = s->parameters.cpu_throttle_increment; + params->has_cpu_throttle_tailslow = true; + params->cpu_throttle_tailslow = s->parameters.cpu_throttle_tailslow; + params->has_tls_creds = true; + params->tls_creds = g_strdup(s->parameters.tls_creds); + params->has_tls_hostname = true; + params->tls_hostname = g_strdup(s->parameters.tls_hostname); + params->has_tls_authz = true; + params->tls_authz = g_strdup(s->parameters.tls_authz ? + s->parameters.tls_authz : ""); + params->has_max_bandwidth = true; + params->max_bandwidth = s->parameters.max_bandwidth; + params->has_downtime_limit = true; + params->downtime_limit = s->parameters.downtime_limit; + params->has_x_checkpoint_delay = true; + params->x_checkpoint_delay = s->parameters.x_checkpoint_delay; + params->has_block_incremental = true; + params->block_incremental = s->parameters.block_incremental; + params->has_multifd_channels = true; + params->multifd_channels = s->parameters.multifd_channels; + params->has_multifd_compression = true; + params->multifd_compression = s->parameters.multifd_compression; + params->has_multifd_zlib_level = true; + params->multifd_zlib_level = s->parameters.multifd_zlib_level; + params->has_multifd_zstd_level = true; + params->multifd_zstd_level = s->parameters.multifd_zstd_level; + params->has_xbzrle_cache_size = true; + params->xbzrle_cache_size = s->parameters.xbzrle_cache_size; + params->has_max_postcopy_bandwidth = true; + params->max_postcopy_bandwidth = s->parameters.max_postcopy_bandwidth; + params->has_max_cpu_throttle = true; + params->max_cpu_throttle = s->parameters.max_cpu_throttle; + params->has_announce_initial = true; + params->announce_initial = s->parameters.announce_initial; + params->has_announce_max = true; + params->announce_max = s->parameters.announce_max; + params->has_announce_rounds = true; + params->announce_rounds = s->parameters.announce_rounds; + params->has_announce_step = true; + params->announce_step = s->parameters.announce_step; + + if (s->parameters.has_block_bitmap_mapping) { + params->has_block_bitmap_mapping = true; + params->block_bitmap_mapping = + QAPI_CLONE(BitmapMigrationNodeAliasList, + s->parameters.block_bitmap_mapping); + } + + return params; +} + +AnnounceParameters *migrate_announce_params(void) +{ + static AnnounceParameters ap; + + MigrationState *s = migrate_get_current(); + + ap.initial = s->parameters.announce_initial; + ap.max = s->parameters.announce_max; + ap.rounds = s->parameters.announce_rounds; + ap.step = s->parameters.announce_step; + + return ≈ +} + +/* + * Return true if we're already in the middle of a migration + * (i.e. any of the active or setup states) + */ +bool migration_is_setup_or_active(int state) +{ + switch (state) { + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_PAUSED: + case MIGRATION_STATUS_POSTCOPY_RECOVER: + case MIGRATION_STATUS_SETUP: + case MIGRATION_STATUS_PRE_SWITCHOVER: + case MIGRATION_STATUS_DEVICE: + case MIGRATION_STATUS_WAIT_UNPLUG: + case MIGRATION_STATUS_COLO: + return true; + + default: + return false; + + } +} + +bool migration_is_running(int state) +{ + switch (state) { + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_PAUSED: + case MIGRATION_STATUS_POSTCOPY_RECOVER: + case MIGRATION_STATUS_SETUP: + case MIGRATION_STATUS_PRE_SWITCHOVER: + case MIGRATION_STATUS_DEVICE: + case MIGRATION_STATUS_WAIT_UNPLUG: + case MIGRATION_STATUS_CANCELLING: + return true; + + default: + return false; + + } +} + +static void populate_time_info(MigrationInfo *info, MigrationState *s) +{ + info->has_status = true; + info->has_setup_time = true; + info->setup_time = s->setup_time; + if (s->state == MIGRATION_STATUS_COMPLETED) { + info->has_total_time = true; + info->total_time = s->total_time; + info->has_downtime = true; + info->downtime = s->downtime; + } else { + info->has_total_time = true; + info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - + s->start_time; + info->has_expected_downtime = true; + info->expected_downtime = s->expected_downtime; + } +} + +static void populate_ram_info(MigrationInfo *info, MigrationState *s) +{ + info->has_ram = true; + info->ram = g_malloc0(sizeof(*info->ram)); + info->ram->transferred = ram_counters.transferred; + info->ram->total = ram_bytes_total(); + info->ram->duplicate = ram_counters.duplicate; + /* legacy value. It is not used anymore */ + info->ram->skipped = 0; + info->ram->normal = ram_counters.normal; + info->ram->normal_bytes = ram_counters.normal * + qemu_target_page_size(); + info->ram->mbps = s->mbps; + info->ram->dirty_sync_count = ram_counters.dirty_sync_count; + info->ram->postcopy_requests = ram_counters.postcopy_requests; + info->ram->page_size = qemu_target_page_size(); + info->ram->multifd_bytes = ram_counters.multifd_bytes; + info->ram->pages_per_second = s->pages_per_second; + + if (migrate_use_xbzrle()) { + info->has_xbzrle_cache = true; + info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache)); + info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size(); + info->xbzrle_cache->bytes = xbzrle_counters.bytes; + info->xbzrle_cache->pages = xbzrle_counters.pages; + info->xbzrle_cache->cache_miss = xbzrle_counters.cache_miss; + info->xbzrle_cache->cache_miss_rate = xbzrle_counters.cache_miss_rate; + info->xbzrle_cache->encoding_rate = xbzrle_counters.encoding_rate; + info->xbzrle_cache->overflow = xbzrle_counters.overflow; + } + + if (migrate_use_compression()) { + info->has_compression = true; + info->compression = g_malloc0(sizeof(*info->compression)); + info->compression->pages = compression_counters.pages; + info->compression->busy = compression_counters.busy; + info->compression->busy_rate = compression_counters.busy_rate; + info->compression->compressed_size = + compression_counters.compressed_size; + info->compression->compression_rate = + compression_counters.compression_rate; + } + + if (cpu_throttle_active()) { + info->has_cpu_throttle_percentage = true; + info->cpu_throttle_percentage = cpu_throttle_get_percentage(); + } + + if (s->state != MIGRATION_STATUS_COMPLETED) { + info->ram->remaining = ram_bytes_remaining(); + info->ram->dirty_pages_rate = ram_counters.dirty_pages_rate; + } +} + +static void populate_disk_info(MigrationInfo *info) +{ + if (blk_mig_active()) { + info->has_disk = true; + info->disk = g_malloc0(sizeof(*info->disk)); + info->disk->transferred = blk_mig_bytes_transferred(); + info->disk->remaining = blk_mig_bytes_remaining(); + info->disk->total = blk_mig_bytes_total(); + } +} + +static void fill_source_migration_info(MigrationInfo *info) +{ + MigrationState *s = migrate_get_current(); + GSList *cur_blocker = migration_blockers; + + info->blocked_reasons = NULL; + + /* + * There are two types of reasons a migration might be blocked; + * a) devices marked in VMState as non-migratable, and + * b) Explicit migration blockers + * We need to add both of them here. + */ + qemu_savevm_non_migratable_list(&info->blocked_reasons); + + while (cur_blocker) { + QAPI_LIST_PREPEND(info->blocked_reasons, + g_strdup(error_get_pretty(cur_blocker->data))); + cur_blocker = g_slist_next(cur_blocker); + } + info->has_blocked_reasons = info->blocked_reasons != NULL; + + switch (s->state) { + case MIGRATION_STATUS_NONE: + /* no migration has happened ever */ + /* do not overwrite destination migration status */ + return; + case MIGRATION_STATUS_SETUP: + info->has_status = true; + info->has_total_time = false; + break; + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_CANCELLING: + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_PRE_SWITCHOVER: + case MIGRATION_STATUS_DEVICE: + case MIGRATION_STATUS_POSTCOPY_PAUSED: + case MIGRATION_STATUS_POSTCOPY_RECOVER: + /* TODO add some postcopy stats */ + populate_time_info(info, s); + populate_ram_info(info, s); + populate_disk_info(info); + populate_vfio_info(info); + break; + case MIGRATION_STATUS_COLO: + info->has_status = true; + /* TODO: display COLO specific information (checkpoint info etc.) */ + break; + case MIGRATION_STATUS_COMPLETED: + populate_time_info(info, s); + populate_ram_info(info, s); + populate_vfio_info(info); + break; + case MIGRATION_STATUS_FAILED: + info->has_status = true; + if (s->error) { + info->has_error_desc = true; + info->error_desc = g_strdup(error_get_pretty(s->error)); + } + break; + case MIGRATION_STATUS_CANCELLED: + info->has_status = true; + break; + case MIGRATION_STATUS_WAIT_UNPLUG: + info->has_status = true; + break; + } + info->status = s->state; +} + +typedef enum WriteTrackingSupport { + WT_SUPPORT_UNKNOWN = 0, + WT_SUPPORT_ABSENT, + WT_SUPPORT_AVAILABLE, + WT_SUPPORT_COMPATIBLE +} WriteTrackingSupport; + +static +WriteTrackingSupport migrate_query_write_tracking(void) +{ + /* Check if kernel supports required UFFD features */ + if (!ram_write_tracking_available()) { + return WT_SUPPORT_ABSENT; + } + /* + * Check if current memory configuration is + * compatible with required UFFD features. + */ + if (!ram_write_tracking_compatible()) { + return WT_SUPPORT_AVAILABLE; + } + + return WT_SUPPORT_COMPATIBLE; +} + +/** + * @migration_caps_check - check capability validity + * + * @cap_list: old capability list, array of bool + * @params: new capabilities to be applied soon + * @errp: set *errp if the check failed, with reason + * + * Returns true if check passed, otherwise false. + */ +static bool migrate_caps_check(bool *cap_list, + MigrationCapabilityStatusList *params, + Error **errp) +{ + MigrationCapabilityStatusList *cap; + bool old_postcopy_cap; + MigrationIncomingState *mis = migration_incoming_get_current(); + + old_postcopy_cap = cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]; + + for (cap = params; cap; cap = cap->next) { + cap_list[cap->value->capability] = cap->value->state; + } + +#ifndef CONFIG_LIVE_BLOCK_MIGRATION + if (cap_list[MIGRATION_CAPABILITY_BLOCK]) { + error_setg(errp, "QEMU compiled without old-style (blk/-b, inc/-i) " + "block migration"); + error_append_hint(errp, "Use drive_mirror+NBD instead.\n"); + return false; + } +#endif + +#ifndef CONFIG_REPLICATION + if (cap_list[MIGRATION_CAPABILITY_X_COLO]) { + error_setg(errp, "QEMU compiled without replication module" + " can't enable COLO"); + error_append_hint(errp, "Please enable replication before COLO.\n"); + return false; + } +#endif + + if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_RAM]) { + /* This check is reasonably expensive, so only when it's being + * set the first time, also it's only the destination that needs + * special support. + */ + if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) && + !postcopy_ram_supported_by_host(mis)) { + /* postcopy_ram_supported_by_host will have emitted a more + * detailed message + */ + error_setg(errp, "Postcopy is not supported"); + return false; + } + + if (cap_list[MIGRATION_CAPABILITY_X_IGNORE_SHARED]) { + error_setg(errp, "Postcopy is not compatible with ignore-shared"); + return false; + } + } + + if (cap_list[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]) { + WriteTrackingSupport wt_support; + int idx; + /* + * Check if 'background-snapshot' capability is supported by + * host kernel and compatible with guest memory configuration. + */ + wt_support = migrate_query_write_tracking(); + if (wt_support < WT_SUPPORT_AVAILABLE) { + error_setg(errp, "Background-snapshot is not supported by host kernel"); + return false; + } + if (wt_support < WT_SUPPORT_COMPATIBLE) { + error_setg(errp, "Background-snapshot is not compatible " + "with guest memory configuration"); + return false; + } + + /* + * Check if there are any migration capabilities + * incompatible with 'background-snapshot'. + */ + for (idx = 0; idx < check_caps_background_snapshot.size; idx++) { + int incomp_cap = check_caps_background_snapshot.caps[idx]; + if (cap_list[incomp_cap]) { + error_setg(errp, + "Background-snapshot is not compatible with %s", + MigrationCapability_str(incomp_cap)); + return false; + } + } + } + + /* incoming side only */ + if (runstate_check(RUN_STATE_INMIGRATE) && + !migrate_multifd_is_allowed() && + cap_list[MIGRATION_CAPABILITY_MULTIFD]) { + error_setg(errp, "multifd is not supported by current protocol"); + return false; + } + + return true; +} + +static void fill_destination_migration_info(MigrationInfo *info) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + if (mis->socket_address_list) { + info->has_socket_address = true; + info->socket_address = + QAPI_CLONE(SocketAddressList, mis->socket_address_list); + } + + switch (mis->state) { + case MIGRATION_STATUS_NONE: + return; + case MIGRATION_STATUS_SETUP: + case MIGRATION_STATUS_CANCELLING: + case MIGRATION_STATUS_CANCELLED: + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_PAUSED: + case MIGRATION_STATUS_POSTCOPY_RECOVER: + case MIGRATION_STATUS_FAILED: + case MIGRATION_STATUS_COLO: + info->has_status = true; + break; + case MIGRATION_STATUS_COMPLETED: + info->has_status = true; + fill_destination_postcopy_migration_info(info); + break; + } + info->status = mis->state; +} + +MigrationInfo *qmp_query_migrate(Error **errp) +{ + MigrationInfo *info = g_malloc0(sizeof(*info)); + + fill_destination_migration_info(info); + fill_source_migration_info(info); + + return info; +} + +void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, + Error **errp) +{ + MigrationState *s = migrate_get_current(); + MigrationCapabilityStatusList *cap; + bool cap_list[MIGRATION_CAPABILITY__MAX]; + + if (migration_is_running(s->state)) { + error_setg(errp, QERR_MIGRATION_ACTIVE); + return; + } + + memcpy(cap_list, s->enabled_capabilities, sizeof(cap_list)); + if (!migrate_caps_check(cap_list, params, errp)) { + return; + } + + for (cap = params; cap; cap = cap->next) { + s->enabled_capabilities[cap->value->capability] = cap->value->state; + } +} + +/* + * Check whether the parameters are valid. Error will be put into errp + * (if provided). Return true if valid, otherwise false. + */ +static bool migrate_params_check(MigrationParameters *params, Error **errp) +{ + if (params->has_compress_level && + (params->compress_level > 9)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level", + "a value between 0 and 9"); + return false; + } + + if (params->has_compress_threads && (params->compress_threads < 1)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "compress_threads", + "a value between 1 and 255"); + return false; + } + + if (params->has_decompress_threads && (params->decompress_threads < 1)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "decompress_threads", + "a value between 1 and 255"); + return false; + } + + if (params->has_throttle_trigger_threshold && + (params->throttle_trigger_threshold < 1 || + params->throttle_trigger_threshold > 100)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "throttle_trigger_threshold", + "an integer in the range of 1 to 100"); + return false; + } + + if (params->has_cpu_throttle_initial && + (params->cpu_throttle_initial < 1 || + params->cpu_throttle_initial > 99)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "cpu_throttle_initial", + "an integer in the range of 1 to 99"); + return false; + } + + if (params->has_cpu_throttle_increment && + (params->cpu_throttle_increment < 1 || + params->cpu_throttle_increment > 99)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "cpu_throttle_increment", + "an integer in the range of 1 to 99"); + return false; + } + + if (params->has_max_bandwidth && (params->max_bandwidth > SIZE_MAX)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "max_bandwidth", + "an integer in the range of 0 to "stringify(SIZE_MAX) + " bytes/second"); + return false; + } + + if (params->has_downtime_limit && + (params->downtime_limit > MAX_MIGRATE_DOWNTIME)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "downtime_limit", + "an integer in the range of 0 to " + stringify(MAX_MIGRATE_DOWNTIME)" ms"); + return false; + } + + /* x_checkpoint_delay is now always positive */ + + if (params->has_multifd_channels && (params->multifd_channels < 1)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "multifd_channels", + "a value between 1 and 255"); + return false; + } + + if (params->has_multifd_zlib_level && + (params->multifd_zlib_level > 9)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zlib_level", + "a value between 0 and 9"); + return false; + } + + if (params->has_multifd_zstd_level && + (params->multifd_zstd_level > 20)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level", + "a value between 0 and 20"); + return false; + } + + if (params->has_xbzrle_cache_size && + (params->xbzrle_cache_size < qemu_target_page_size() || + !is_power_of_2(params->xbzrle_cache_size))) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "xbzrle_cache_size", + "a power of two no less than the target page size"); + return false; + } + + if (params->has_max_cpu_throttle && + (params->max_cpu_throttle < params->cpu_throttle_initial || + params->max_cpu_throttle > 99)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "max_cpu_throttle", + "an integer in the range of cpu_throttle_initial to 99"); + return false; + } + + if (params->has_announce_initial && + params->announce_initial > 100000) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "announce_initial", + "a value between 0 and 100000"); + return false; + } + if (params->has_announce_max && + params->announce_max > 100000) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "announce_max", + "a value between 0 and 100000"); + return false; + } + if (params->has_announce_rounds && + params->announce_rounds > 1000) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "announce_rounds", + "a value between 0 and 1000"); + return false; + } + if (params->has_announce_step && + (params->announce_step < 1 || + params->announce_step > 10000)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "announce_step", + "a value between 0 and 10000"); + return false; + } + + if (params->has_block_bitmap_mapping && + !check_dirty_bitmap_mig_alias_map(params->block_bitmap_mapping, errp)) { + error_prepend(errp, "Invalid mapping given for block-bitmap-mapping: "); + return false; + } + + return true; +} + +static void migrate_params_test_apply(MigrateSetParameters *params, + MigrationParameters *dest) +{ + *dest = migrate_get_current()->parameters; + + /* TODO use QAPI_CLONE() instead of duplicating it inline */ + + if (params->has_compress_level) { + dest->compress_level = params->compress_level; + } + + if (params->has_compress_threads) { + dest->compress_threads = params->compress_threads; + } + + if (params->has_compress_wait_thread) { + dest->compress_wait_thread = params->compress_wait_thread; + } + + if (params->has_decompress_threads) { + dest->decompress_threads = params->decompress_threads; + } + + if (params->has_throttle_trigger_threshold) { + dest->throttle_trigger_threshold = params->throttle_trigger_threshold; + } + + if (params->has_cpu_throttle_initial) { + dest->cpu_throttle_initial = params->cpu_throttle_initial; + } + + if (params->has_cpu_throttle_increment) { + dest->cpu_throttle_increment = params->cpu_throttle_increment; + } + + if (params->has_cpu_throttle_tailslow) { + dest->cpu_throttle_tailslow = params->cpu_throttle_tailslow; + } + + if (params->has_tls_creds) { + assert(params->tls_creds->type == QTYPE_QSTRING); + dest->tls_creds = params->tls_creds->u.s; + } + + if (params->has_tls_hostname) { + assert(params->tls_hostname->type == QTYPE_QSTRING); + dest->tls_hostname = params->tls_hostname->u.s; + } + + if (params->has_max_bandwidth) { + dest->max_bandwidth = params->max_bandwidth; + } + + if (params->has_downtime_limit) { + dest->downtime_limit = params->downtime_limit; + } + + if (params->has_x_checkpoint_delay) { + dest->x_checkpoint_delay = params->x_checkpoint_delay; + } + + if (params->has_block_incremental) { + dest->block_incremental = params->block_incremental; + } + if (params->has_multifd_channels) { + dest->multifd_channels = params->multifd_channels; + } + if (params->has_multifd_compression) { + dest->multifd_compression = params->multifd_compression; + } + if (params->has_xbzrle_cache_size) { + dest->xbzrle_cache_size = params->xbzrle_cache_size; + } + if (params->has_max_postcopy_bandwidth) { + dest->max_postcopy_bandwidth = params->max_postcopy_bandwidth; + } + if (params->has_max_cpu_throttle) { + dest->max_cpu_throttle = params->max_cpu_throttle; + } + if (params->has_announce_initial) { + dest->announce_initial = params->announce_initial; + } + if (params->has_announce_max) { + dest->announce_max = params->announce_max; + } + if (params->has_announce_rounds) { + dest->announce_rounds = params->announce_rounds; + } + if (params->has_announce_step) { + dest->announce_step = params->announce_step; + } + + if (params->has_block_bitmap_mapping) { + dest->has_block_bitmap_mapping = true; + dest->block_bitmap_mapping = params->block_bitmap_mapping; + } +} + +static void migrate_params_apply(MigrateSetParameters *params, Error **errp) +{ + MigrationState *s = migrate_get_current(); + + /* TODO use QAPI_CLONE() instead of duplicating it inline */ + + if (params->has_compress_level) { + s->parameters.compress_level = params->compress_level; + } + + if (params->has_compress_threads) { + s->parameters.compress_threads = params->compress_threads; + } + + if (params->has_compress_wait_thread) { + s->parameters.compress_wait_thread = params->compress_wait_thread; + } + + if (params->has_decompress_threads) { + s->parameters.decompress_threads = params->decompress_threads; + } + + if (params->has_throttle_trigger_threshold) { + s->parameters.throttle_trigger_threshold = params->throttle_trigger_threshold; + } + + if (params->has_cpu_throttle_initial) { + s->parameters.cpu_throttle_initial = params->cpu_throttle_initial; + } + + if (params->has_cpu_throttle_increment) { + s->parameters.cpu_throttle_increment = params->cpu_throttle_increment; + } + + if (params->has_cpu_throttle_tailslow) { + s->parameters.cpu_throttle_tailslow = params->cpu_throttle_tailslow; + } + + if (params->has_tls_creds) { + g_free(s->parameters.tls_creds); + assert(params->tls_creds->type == QTYPE_QSTRING); + s->parameters.tls_creds = g_strdup(params->tls_creds->u.s); + } + + if (params->has_tls_hostname) { + g_free(s->parameters.tls_hostname); + assert(params->tls_hostname->type == QTYPE_QSTRING); + s->parameters.tls_hostname = g_strdup(params->tls_hostname->u.s); + } + + if (params->has_tls_authz) { + g_free(s->parameters.tls_authz); + assert(params->tls_authz->type == QTYPE_QSTRING); + s->parameters.tls_authz = g_strdup(params->tls_authz->u.s); + } + + if (params->has_max_bandwidth) { + s->parameters.max_bandwidth = params->max_bandwidth; + if (s->to_dst_file && !migration_in_postcopy()) { + qemu_file_set_rate_limit(s->to_dst_file, + s->parameters.max_bandwidth / XFER_LIMIT_RATIO); + } + } + + if (params->has_downtime_limit) { + s->parameters.downtime_limit = params->downtime_limit; + } + + if (params->has_x_checkpoint_delay) { + s->parameters.x_checkpoint_delay = params->x_checkpoint_delay; + if (migration_in_colo_state()) { + colo_checkpoint_notify(s); + } + } + + if (params->has_block_incremental) { + s->parameters.block_incremental = params->block_incremental; + } + if (params->has_multifd_channels) { + s->parameters.multifd_channels = params->multifd_channels; + } + if (params->has_multifd_compression) { + s->parameters.multifd_compression = params->multifd_compression; + } + if (params->has_xbzrle_cache_size) { + s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; + xbzrle_cache_resize(params->xbzrle_cache_size, errp); + } + if (params->has_max_postcopy_bandwidth) { + s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth; + if (s->to_dst_file && migration_in_postcopy()) { + qemu_file_set_rate_limit(s->to_dst_file, + s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO); + } + } + if (params->has_max_cpu_throttle) { + s->parameters.max_cpu_throttle = params->max_cpu_throttle; + } + if (params->has_announce_initial) { + s->parameters.announce_initial = params->announce_initial; + } + if (params->has_announce_max) { + s->parameters.announce_max = params->announce_max; + } + if (params->has_announce_rounds) { + s->parameters.announce_rounds = params->announce_rounds; + } + if (params->has_announce_step) { + s->parameters.announce_step = params->announce_step; + } + + if (params->has_block_bitmap_mapping) { + qapi_free_BitmapMigrationNodeAliasList( + s->parameters.block_bitmap_mapping); + + s->parameters.has_block_bitmap_mapping = true; + s->parameters.block_bitmap_mapping = + QAPI_CLONE(BitmapMigrationNodeAliasList, + params->block_bitmap_mapping); + } +} + +void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp) +{ + MigrationParameters tmp; + + /* TODO Rewrite "" to null instead */ + if (params->has_tls_creds + && params->tls_creds->type == QTYPE_QNULL) { + qobject_unref(params->tls_creds->u.n); + params->tls_creds->type = QTYPE_QSTRING; + params->tls_creds->u.s = strdup(""); + } + /* TODO Rewrite "" to null instead */ + if (params->has_tls_hostname + && params->tls_hostname->type == QTYPE_QNULL) { + qobject_unref(params->tls_hostname->u.n); + params->tls_hostname->type = QTYPE_QSTRING; + params->tls_hostname->u.s = strdup(""); + } + + migrate_params_test_apply(params, &tmp); + + if (!migrate_params_check(&tmp, errp)) { + /* Invalid parameter */ + return; + } + + migrate_params_apply(params, errp); +} + + +void qmp_migrate_start_postcopy(Error **errp) +{ + MigrationState *s = migrate_get_current(); + + if (!migrate_postcopy()) { + error_setg(errp, "Enable postcopy with migrate_set_capability before" + " the start of migration"); + return; + } + + if (s->state == MIGRATION_STATUS_NONE) { + error_setg(errp, "Postcopy must be started after migration has been" + " started"); + return; + } + /* + * we don't error if migration has finished since that would be racy + * with issuing this command. + */ + qatomic_set(&s->start_postcopy, true); +} + +/* shared migration helpers */ + +void migrate_set_state(int *state, int old_state, int new_state) +{ + assert(new_state < MIGRATION_STATUS__MAX); + if (qatomic_cmpxchg(state, old_state, new_state) == old_state) { + trace_migrate_set_state(MigrationStatus_str(new_state)); + migrate_generate_event(new_state); + } +} + +static MigrationCapabilityStatus *migrate_cap_add(MigrationCapability index, + bool state) +{ + MigrationCapabilityStatus *cap; + + cap = g_new0(MigrationCapabilityStatus, 1); + cap->capability = index; + cap->state = state; + + return cap; +} + +void migrate_set_block_enabled(bool value, Error **errp) +{ + MigrationCapabilityStatusList *cap = NULL; + + QAPI_LIST_PREPEND(cap, migrate_cap_add(MIGRATION_CAPABILITY_BLOCK, value)); + qmp_migrate_set_capabilities(cap, errp); + qapi_free_MigrationCapabilityStatusList(cap); +} + +static void migrate_set_block_incremental(MigrationState *s, bool value) +{ + s->parameters.block_incremental = value; +} + +static void block_cleanup_parameters(MigrationState *s) +{ + if (s->must_remove_block_options) { + /* setting to false can never fail */ + migrate_set_block_enabled(false, &error_abort); + migrate_set_block_incremental(s, false); + s->must_remove_block_options = false; + } +} + +static void migrate_fd_cleanup(MigrationState *s) +{ + qemu_bh_delete(s->cleanup_bh); + s->cleanup_bh = NULL; + + qemu_savevm_state_cleanup(); + + if (s->to_dst_file) { + QEMUFile *tmp; + + trace_migrate_fd_cleanup(); + qemu_mutex_unlock_iothread(); + if (s->migration_thread_running) { + qemu_thread_join(&s->thread); + s->migration_thread_running = false; + } + qemu_mutex_lock_iothread(); + + multifd_save_cleanup(); + qemu_mutex_lock(&s->qemu_file_lock); + tmp = s->to_dst_file; + s->to_dst_file = NULL; + qemu_mutex_unlock(&s->qemu_file_lock); + /* + * Close the file handle without the lock to make sure the + * critical section won't block for long. + */ + migration_ioc_unregister_yank_from_file(tmp); + qemu_fclose(tmp); + } + + assert(!migration_is_active(s)); + + if (s->state == MIGRATION_STATUS_CANCELLING) { + migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING, + MIGRATION_STATUS_CANCELLED); + } + + if (s->error) { + /* It is used on info migrate. We can't free it */ + error_report_err(error_copy(s->error)); + } + notifier_list_notify(&migration_state_notifiers, s); + block_cleanup_parameters(s); + yank_unregister_instance(MIGRATION_YANK_INSTANCE); +} + +static void migrate_fd_cleanup_schedule(MigrationState *s) +{ + /* + * Ref the state for bh, because it may be called when + * there're already no other refs + */ + object_ref(OBJECT(s)); + qemu_bh_schedule(s->cleanup_bh); +} + +static void migrate_fd_cleanup_bh(void *opaque) +{ + MigrationState *s = opaque; + migrate_fd_cleanup(s); + object_unref(OBJECT(s)); +} + +void migrate_set_error(MigrationState *s, const Error *error) +{ + QEMU_LOCK_GUARD(&s->error_mutex); + if (!s->error) { + s->error = error_copy(error); + } +} + +static void migrate_error_free(MigrationState *s) +{ + QEMU_LOCK_GUARD(&s->error_mutex); + if (s->error) { + error_free(s->error); + s->error = NULL; + } +} + +void migrate_fd_error(MigrationState *s, const Error *error) +{ + trace_migrate_fd_error(error_get_pretty(error)); + assert(s->to_dst_file == NULL); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); + migrate_set_error(s, error); +} + +static void migrate_fd_cancel(MigrationState *s) +{ + int old_state ; + QEMUFile *f = migrate_get_current()->to_dst_file; + trace_migrate_fd_cancel(); + + WITH_QEMU_LOCK_GUARD(&s->qemu_file_lock) { + if (s->rp_state.from_dst_file) { + /* shutdown the rp socket, so causing the rp thread to shutdown */ + qemu_file_shutdown(s->rp_state.from_dst_file); + } + } + + do { + old_state = s->state; + if (!migration_is_running(old_state)) { + break; + } + /* If the migration is paused, kick it out of the pause */ + if (old_state == MIGRATION_STATUS_PRE_SWITCHOVER) { + qemu_sem_post(&s->pause_sem); + } + migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING); + } while (s->state != MIGRATION_STATUS_CANCELLING); + + /* + * If we're unlucky the migration code might be stuck somewhere in a + * send/write while the network has failed and is waiting to timeout; + * if we've got shutdown(2) available then we can force it to quit. + * The outgoing qemu file gets closed in migrate_fd_cleanup that is + * called in a bh, so there is no race against this cancel. + */ + if (s->state == MIGRATION_STATUS_CANCELLING && f) { + qemu_file_shutdown(f); + } + if (s->state == MIGRATION_STATUS_CANCELLING && s->block_inactive) { + Error *local_err = NULL; + + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + } else { + s->block_inactive = false; + } + } +} + +void add_migration_state_change_notifier(Notifier *notify) +{ + notifier_list_add(&migration_state_notifiers, notify); +} + +void remove_migration_state_change_notifier(Notifier *notify) +{ + notifier_remove(notify); +} + +bool migration_in_setup(MigrationState *s) +{ + return s->state == MIGRATION_STATUS_SETUP; +} + +bool migration_has_finished(MigrationState *s) +{ + return s->state == MIGRATION_STATUS_COMPLETED; +} + +bool migration_has_failed(MigrationState *s) +{ + return (s->state == MIGRATION_STATUS_CANCELLED || + s->state == MIGRATION_STATUS_FAILED); +} + +bool migration_in_postcopy(void) +{ + MigrationState *s = migrate_get_current(); + + switch (s->state) { + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_PAUSED: + case MIGRATION_STATUS_POSTCOPY_RECOVER: + return true; + default: + return false; + } +} + +bool migration_in_postcopy_after_devices(MigrationState *s) +{ + return migration_in_postcopy() && s->postcopy_after_devices; +} + +bool migration_in_incoming_postcopy(void) +{ + PostcopyState ps = postcopy_state_get(); + + return ps >= POSTCOPY_INCOMING_DISCARD && ps < POSTCOPY_INCOMING_END; +} + +bool migration_in_bg_snapshot(void) +{ + MigrationState *s = migrate_get_current(); + + return migrate_background_snapshot() && + migration_is_setup_or_active(s->state); +} + +bool migration_is_idle(void) +{ + MigrationState *s = current_migration; + + if (!s) { + return true; + } + + switch (s->state) { + case MIGRATION_STATUS_NONE: + case MIGRATION_STATUS_CANCELLED: + case MIGRATION_STATUS_COMPLETED: + case MIGRATION_STATUS_FAILED: + return true; + case MIGRATION_STATUS_SETUP: + case MIGRATION_STATUS_CANCELLING: + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_COLO: + case MIGRATION_STATUS_PRE_SWITCHOVER: + case MIGRATION_STATUS_DEVICE: + case MIGRATION_STATUS_WAIT_UNPLUG: + return false; + case MIGRATION_STATUS__MAX: + g_assert_not_reached(); + } + + return false; +} + +bool migration_is_active(MigrationState *s) +{ + return (s->state == MIGRATION_STATUS_ACTIVE || + s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE); +} + +void migrate_init(MigrationState *s) +{ + /* + * Reinitialise all migration state, except + * parameters/capabilities that the user set, and + * locks. + */ + s->cleanup_bh = 0; + s->vm_start_bh = 0; + s->to_dst_file = NULL; + s->state = MIGRATION_STATUS_NONE; + s->rp_state.from_dst_file = NULL; + s->rp_state.error = false; + s->mbps = 0.0; + s->pages_per_second = 0.0; + s->downtime = 0; + s->expected_downtime = 0; + s->setup_time = 0; + s->start_postcopy = false; + s->postcopy_after_devices = false; + s->migration_thread_running = false; + error_free(s->error); + s->error = NULL; + s->hostname = NULL; + + migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); + + s->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + s->total_time = 0; + s->vm_was_running = false; + s->iteration_initial_bytes = 0; + s->threshold_size = 0; +} + +int migrate_add_blocker_internal(Error *reason, Error **errp) +{ + /* Snapshots are similar to migrations, so check RUN_STATE_SAVE_VM too. */ + if (runstate_check(RUN_STATE_SAVE_VM) || !migration_is_idle()) { + error_propagate_prepend(errp, error_copy(reason), + "disallowing migration blocker " + "(migration/snapshot in progress) for: "); + return -EBUSY; + } + + migration_blockers = g_slist_prepend(migration_blockers, reason); + return 0; +} + +int migrate_add_blocker(Error *reason, Error **errp) +{ + if (only_migratable) { + error_propagate_prepend(errp, error_copy(reason), + "disallowing migration blocker " + "(--only-migratable) for: "); + return -EACCES; + } + + return migrate_add_blocker_internal(reason, errp); +} + +void migrate_del_blocker(Error *reason) +{ + migration_blockers = g_slist_remove(migration_blockers, reason); +} + +void qmp_migrate_incoming(const char *uri, Error **errp) +{ + Error *local_err = NULL; + static bool once = true; + + if (!once) { + error_setg(errp, "The incoming migration has already been started"); + return; + } + if (!runstate_check(RUN_STATE_INMIGRATE)) { + error_setg(errp, "'-incoming' was not specified on the command line"); + return; + } + + if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) { + return; + } + + qemu_start_incoming_migration(uri, &local_err); + + if (local_err) { + yank_unregister_instance(MIGRATION_YANK_INSTANCE); + error_propagate(errp, local_err); + return; + } + + once = false; +} + +void qmp_migrate_recover(const char *uri, Error **errp) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + /* + * Don't even bother to use ERRP_GUARD() as it _must_ always be set by + * callers (no one should ignore a recover failure); if there is, it's a + * programming error. + */ + assert(errp); + + if (mis->state != MIGRATION_STATUS_POSTCOPY_PAUSED) { + error_setg(errp, "Migrate recover can only be run " + "when postcopy is paused."); + return; + } + + if (qatomic_cmpxchg(&mis->postcopy_recover_triggered, + false, true) == true) { + error_setg(errp, "Migrate recovery is triggered already"); + return; + } + + /* + * Note that this call will never start a real migration; it will + * only re-setup the migration stream and poke existing migration + * to continue using that newly established channel. + */ + qemu_start_incoming_migration(uri, errp); + + /* Safe to dereference with the assert above */ + if (*errp) { + /* Reset the flag so user could still retry */ + qatomic_set(&mis->postcopy_recover_triggered, false); + } +} + +void qmp_migrate_pause(Error **errp) +{ + MigrationState *ms = migrate_get_current(); + MigrationIncomingState *mis = migration_incoming_get_current(); + int ret; + + if (ms->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) { + /* Source side, during postcopy */ + qemu_mutex_lock(&ms->qemu_file_lock); + ret = qemu_file_shutdown(ms->to_dst_file); + qemu_mutex_unlock(&ms->qemu_file_lock); + if (ret) { + error_setg(errp, "Failed to pause source migration"); + } + return; + } + + if (mis->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) { + ret = qemu_file_shutdown(mis->from_src_file); + if (ret) { + error_setg(errp, "Failed to pause destination migration"); + } + return; + } + + error_setg(errp, "migrate-pause is currently only supported " + "during postcopy-active state"); +} + +bool migration_is_blocked(Error **errp) +{ + if (qemu_savevm_state_blocked(errp)) { + return true; + } + + if (migration_blockers) { + error_propagate(errp, error_copy(migration_blockers->data)); + return true; + } + + return false; +} + +/* Returns true if continue to migrate, or false if error detected */ +static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, + bool resume, Error **errp) +{ + Error *local_err = NULL; + + if (resume) { + if (s->state != MIGRATION_STATUS_POSTCOPY_PAUSED) { + error_setg(errp, "Cannot resume if there is no " + "paused migration"); + return false; + } + + /* + * Postcopy recovery won't work well with release-ram + * capability since release-ram will drop the page buffer as + * long as the page is put into the send buffer. So if there + * is a network failure happened, any page buffers that have + * not yet reached the destination VM but have already been + * sent from the source VM will be lost forever. Let's refuse + * the client from resuming such a postcopy migration. + * Luckily release-ram was designed to only be used when src + * and destination VMs are on the same host, so it should be + * fine. + */ + if (migrate_release_ram()) { + error_setg(errp, "Postcopy recovery cannot work " + "when release-ram capability is set"); + return false; + } + + /* This is a resume, skip init status */ + return true; + } + + if (migration_is_running(s->state)) { + error_setg(errp, QERR_MIGRATION_ACTIVE); + return false; + } + + if (runstate_check(RUN_STATE_INMIGRATE)) { + error_setg(errp, "Guest is waiting for an incoming migration"); + return false; + } + + if (runstate_check(RUN_STATE_POSTMIGRATE)) { + error_setg(errp, "Can't migrate the vm that was paused due to " + "previous migration"); + return false; + } + + if (migration_is_blocked(errp)) { + return false; + } + + if (blk || blk_inc) { + if (migrate_colo_enabled()) { + error_setg(errp, "No disk migration is required in COLO mode"); + return false; + } + if (migrate_use_block() || migrate_use_block_incremental()) { + error_setg(errp, "Command options are incompatible with " + "current migration capabilities"); + return false; + } + migrate_set_block_enabled(true, &local_err); + if (local_err) { + error_propagate(errp, local_err); + return false; + } + s->must_remove_block_options = true; + } + + if (blk_inc) { + migrate_set_block_incremental(s, true); + } + + migrate_init(s); + /* + * set ram_counters compression_counters memory to zero for a + * new migration + */ + memset(&ram_counters, 0, sizeof(ram_counters)); + memset(&compression_counters, 0, sizeof(compression_counters)); + + return true; +} + +void qmp_migrate(const char *uri, bool has_blk, bool blk, + bool has_inc, bool inc, bool has_detach, bool detach, + bool has_resume, bool resume, Error **errp) +{ + Error *local_err = NULL; + MigrationState *s = migrate_get_current(); + const char *p = NULL; + + if (!migrate_prepare(s, has_blk && blk, has_inc && inc, + has_resume && resume, errp)) { + /* Error detected, put into errp */ + return; + } + + if (!(has_resume && resume)) { + if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) { + return; + } + } + + migrate_protocol_allow_multifd(false); + if (strstart(uri, "tcp:", &p) || + strstart(uri, "unix:", NULL) || + strstart(uri, "vsock:", NULL)) { + migrate_protocol_allow_multifd(true); + socket_start_outgoing_migration(s, p ? p : uri, &local_err); +#ifdef CONFIG_RDMA + } else if (strstart(uri, "rdma:", &p)) { + rdma_start_outgoing_migration(s, p, &local_err); +#endif + } else if (strstart(uri, "exec:", &p)) { + exec_start_outgoing_migration(s, p, &local_err); + } else if (strstart(uri, "fd:", &p)) { + fd_start_outgoing_migration(s, p, &local_err); + } else { + if (!(has_resume && resume)) { + yank_unregister_instance(MIGRATION_YANK_INSTANCE); + } + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri", + "a valid migration protocol"); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); + block_cleanup_parameters(s); + return; + } + + if (local_err) { + if (!(has_resume && resume)) { + yank_unregister_instance(MIGRATION_YANK_INSTANCE); + } + migrate_fd_error(s, local_err); + error_propagate(errp, local_err); + return; + } +} + +void qmp_migrate_cancel(Error **errp) +{ + migration_cancel(NULL); +} + +void qmp_migrate_continue(MigrationStatus state, Error **errp) +{ + MigrationState *s = migrate_get_current(); + if (s->state != state) { + error_setg(errp, "Migration not in expected state: %s", + MigrationStatus_str(s->state)); + return; + } + qemu_sem_post(&s->pause_sem); +} + +bool migrate_release_ram(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_RELEASE_RAM]; +} + +bool migrate_postcopy_ram(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM]; +} + +bool migrate_postcopy(void) +{ + return migrate_postcopy_ram() || migrate_dirty_bitmaps(); +} + +bool migrate_auto_converge(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE]; +} + +bool migrate_zero_blocks(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS]; +} + +bool migrate_postcopy_blocktime(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_BLOCKTIME]; +} + +bool migrate_use_compression(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS]; +} + +int migrate_compress_level(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.compress_level; +} + +int migrate_compress_threads(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.compress_threads; +} + +int migrate_compress_wait_thread(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.compress_wait_thread; +} + +int migrate_decompress_threads(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.decompress_threads; +} + +bool migrate_dirty_bitmaps(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_DIRTY_BITMAPS]; +} + +bool migrate_ignore_shared(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_X_IGNORE_SHARED]; +} + +bool migrate_validate_uuid(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_VALIDATE_UUID]; +} + +bool migrate_use_events(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS]; +} + +bool migrate_use_multifd(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_MULTIFD]; +} + +bool migrate_pause_before_switchover(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[ + MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER]; +} + +int migrate_multifd_channels(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.multifd_channels; +} + +MultiFDCompression migrate_multifd_compression(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.multifd_compression; +} + +int migrate_multifd_zlib_level(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.multifd_zlib_level; +} + +int migrate_multifd_zstd_level(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.multifd_zstd_level; +} + +int migrate_use_xbzrle(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE]; +} + +uint64_t migrate_xbzrle_cache_size(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.xbzrle_cache_size; +} + +static int64_t migrate_max_postcopy_bandwidth(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.max_postcopy_bandwidth; +} + +bool migrate_use_block(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_BLOCK]; +} + +bool migrate_use_return_path(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_RETURN_PATH]; +} + +bool migrate_use_block_incremental(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->parameters.block_incremental; +} + +bool migrate_background_snapshot(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT]; +} + +/* migration thread support */ +/* + * Something bad happened to the RP stream, mark an error + * The caller shall print or trace something to indicate why + */ +static void mark_source_rp_bad(MigrationState *s) +{ + s->rp_state.error = true; +} + +static struct rp_cmd_args { + ssize_t len; /* -1 = variable */ + const char *name; +} rp_cmd_args[] = { + [MIG_RP_MSG_INVALID] = { .len = -1, .name = "INVALID" }, + [MIG_RP_MSG_SHUT] = { .len = 4, .name = "SHUT" }, + [MIG_RP_MSG_PONG] = { .len = 4, .name = "PONG" }, + [MIG_RP_MSG_REQ_PAGES] = { .len = 12, .name = "REQ_PAGES" }, + [MIG_RP_MSG_REQ_PAGES_ID] = { .len = -1, .name = "REQ_PAGES_ID" }, + [MIG_RP_MSG_RECV_BITMAP] = { .len = -1, .name = "RECV_BITMAP" }, + [MIG_RP_MSG_RESUME_ACK] = { .len = 4, .name = "RESUME_ACK" }, + [MIG_RP_MSG_MAX] = { .len = -1, .name = "MAX" }, +}; + +/* + * Process a request for pages received on the return path, + * We're allowed to send more than requested (e.g. to round to our page size) + * and we don't need to send pages that have already been sent. + */ +static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname, + ram_addr_t start, size_t len) +{ + long our_host_ps = qemu_real_host_page_size; + + trace_migrate_handle_rp_req_pages(rbname, start, len); + + /* + * Since we currently insist on matching page sizes, just sanity check + * we're being asked for whole host pages. + */ + if (!QEMU_IS_ALIGNED(start, our_host_ps) || + !QEMU_IS_ALIGNED(len, our_host_ps)) { + error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT + " len: %zd", __func__, start, len); + mark_source_rp_bad(ms); + return; + } + + if (ram_save_queue_pages(rbname, start, len)) { + mark_source_rp_bad(ms); + } +} + +/* Return true to retry, false to quit */ +static bool postcopy_pause_return_path_thread(MigrationState *s) +{ + trace_postcopy_pause_return_path(); + + qemu_sem_wait(&s->postcopy_pause_rp_sem); + + trace_postcopy_pause_return_path_continued(); + + return true; +} + +static int migrate_handle_rp_recv_bitmap(MigrationState *s, char *block_name) +{ + RAMBlock *block = qemu_ram_block_by_name(block_name); + + if (!block) { + error_report("%s: invalid block name '%s'", __func__, block_name); + return -EINVAL; + } + + /* Fetch the received bitmap and refresh the dirty bitmap */ + return ram_dirty_bitmap_reload(s, block); +} + +static int migrate_handle_rp_resume_ack(MigrationState *s, uint32_t value) +{ + trace_source_return_path_thread_resume_ack(value); + + if (value != MIGRATION_RESUME_ACK_VALUE) { + error_report("%s: illegal resume_ack value %"PRIu32, + __func__, value); + return -1; + } + + /* Now both sides are active. */ + migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_RECOVER, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + + /* Notify send thread that time to continue send pages */ + qemu_sem_post(&s->rp_state.rp_sem); + + return 0; +} + +/* Release ms->rp_state.from_dst_file in a safe way */ +static void migration_release_from_dst_file(MigrationState *ms) +{ + QEMUFile *file; + + WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) { + /* + * Reset the from_dst_file pointer first before releasing it, as we + * can't block within lock section + */ + file = ms->rp_state.from_dst_file; + ms->rp_state.from_dst_file = NULL; + } + + qemu_fclose(file); +} + +/* + * Handles messages sent on the return path towards the source VM + * + */ +static void *source_return_path_thread(void *opaque) +{ + MigrationState *ms = opaque; + QEMUFile *rp = ms->rp_state.from_dst_file; + uint16_t header_len, header_type; + uint8_t buf[512]; + uint32_t tmp32, sibling_error; + ram_addr_t start = 0; /* =0 to silence warning */ + size_t len = 0, expected_len; + int res; + + trace_source_return_path_thread_entry(); + rcu_register_thread(); + +retry: + while (!ms->rp_state.error && !qemu_file_get_error(rp) && + migration_is_setup_or_active(ms->state)) { + trace_source_return_path_thread_loop_top(); + header_type = qemu_get_be16(rp); + header_len = qemu_get_be16(rp); + + if (qemu_file_get_error(rp)) { + mark_source_rp_bad(ms); + goto out; + } + + if (header_type >= MIG_RP_MSG_MAX || + header_type == MIG_RP_MSG_INVALID) { + error_report("RP: Received invalid message 0x%04x length 0x%04x", + header_type, header_len); + mark_source_rp_bad(ms); + goto out; + } + + if ((rp_cmd_args[header_type].len != -1 && + header_len != rp_cmd_args[header_type].len) || + header_len > sizeof(buf)) { + error_report("RP: Received '%s' message (0x%04x) with" + "incorrect length %d expecting %zu", + rp_cmd_args[header_type].name, header_type, header_len, + (size_t)rp_cmd_args[header_type].len); + mark_source_rp_bad(ms); + goto out; + } + + /* We know we've got a valid header by this point */ + res = qemu_get_buffer(rp, buf, header_len); + if (res != header_len) { + error_report("RP: Failed reading data for message 0x%04x" + " read %d expected %d", + header_type, res, header_len); + mark_source_rp_bad(ms); + goto out; + } + + /* OK, we have the message and the data */ + switch (header_type) { + case MIG_RP_MSG_SHUT: + sibling_error = ldl_be_p(buf); + trace_source_return_path_thread_shut(sibling_error); + if (sibling_error) { + error_report("RP: Sibling indicated error %d", sibling_error); + mark_source_rp_bad(ms); + } + /* + * We'll let the main thread deal with closing the RP + * we could do a shutdown(2) on it, but we're the only user + * anyway, so there's nothing gained. + */ + goto out; + + case MIG_RP_MSG_PONG: + tmp32 = ldl_be_p(buf); + trace_source_return_path_thread_pong(tmp32); + break; + + case MIG_RP_MSG_REQ_PAGES: + start = ldq_be_p(buf); + len = ldl_be_p(buf + 8); + migrate_handle_rp_req_pages(ms, NULL, start, len); + break; + + case MIG_RP_MSG_REQ_PAGES_ID: + expected_len = 12 + 1; /* header + termination */ + + if (header_len >= expected_len) { + start = ldq_be_p(buf); + len = ldl_be_p(buf + 8); + /* Now we expect an idstr */ + tmp32 = buf[12]; /* Length of the following idstr */ + buf[13 + tmp32] = '\0'; + expected_len += tmp32; + } + if (header_len != expected_len) { + error_report("RP: Req_Page_id with length %d expecting %zd", + header_len, expected_len); + mark_source_rp_bad(ms); + goto out; + } + migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len); + break; + + case MIG_RP_MSG_RECV_BITMAP: + if (header_len < 1) { + error_report("%s: missing block name", __func__); + mark_source_rp_bad(ms); + goto out; + } + /* Format: len (1B) + idstr (<255B). This ends the idstr. */ + buf[buf[0] + 1] = '\0'; + if (migrate_handle_rp_recv_bitmap(ms, (char *)(buf + 1))) { + mark_source_rp_bad(ms); + goto out; + } + break; + + case MIG_RP_MSG_RESUME_ACK: + tmp32 = ldl_be_p(buf); + if (migrate_handle_rp_resume_ack(ms, tmp32)) { + mark_source_rp_bad(ms); + goto out; + } + break; + + default: + break; + } + } + +out: + res = qemu_file_get_error(rp); + if (res) { + if (res == -EIO && migration_in_postcopy()) { + /* + * Maybe there is something we can do: it looks like a + * network down issue, and we pause for a recovery. + */ + migration_release_from_dst_file(ms); + rp = NULL; + if (postcopy_pause_return_path_thread(ms)) { + /* + * Reload rp, reset the rest. Referencing it is safe since + * it's reset only by us above, or when migration completes + */ + rp = ms->rp_state.from_dst_file; + ms->rp_state.error = false; + goto retry; + } + } + + trace_source_return_path_thread_bad_end(); + mark_source_rp_bad(ms); + } + + trace_source_return_path_thread_end(); + migration_release_from_dst_file(ms); + rcu_unregister_thread(); + return NULL; +} + +static int open_return_path_on_source(MigrationState *ms, + bool create_thread) +{ + ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file); + if (!ms->rp_state.from_dst_file) { + return -1; + } + + trace_open_return_path_on_source(); + + if (!create_thread) { + /* We're done */ + return 0; + } + + qemu_thread_create(&ms->rp_state.rp_thread, "return path", + source_return_path_thread, ms, QEMU_THREAD_JOINABLE); + ms->rp_state.rp_thread_created = true; + + trace_open_return_path_on_source_continue(); + + return 0; +} + +/* Returns 0 if the RP was ok, otherwise there was an error on the RP */ +static int await_return_path_close_on_source(MigrationState *ms) +{ + /* + * If this is a normal exit then the destination will send a SHUT and the + * rp_thread will exit, however if there's an error we need to cause + * it to exit. + */ + if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) { + /* + * shutdown(2), if we have it, will cause it to unblock if it's stuck + * waiting for the destination. + */ + qemu_file_shutdown(ms->rp_state.from_dst_file); + mark_source_rp_bad(ms); + } + trace_await_return_path_close_on_source_joining(); + qemu_thread_join(&ms->rp_state.rp_thread); + ms->rp_state.rp_thread_created = false; + trace_await_return_path_close_on_source_close(); + return ms->rp_state.error; +} + +/* + * Switch from normal iteration to postcopy + * Returns non-0 on error + */ +static int postcopy_start(MigrationState *ms) +{ + int ret; + QIOChannelBuffer *bioc; + QEMUFile *fb; + int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + int64_t bandwidth = migrate_max_postcopy_bandwidth(); + bool restart_block = false; + int cur_state = MIGRATION_STATUS_ACTIVE; + if (!migrate_pause_before_switchover()) { + migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + } + + trace_postcopy_start(); + qemu_mutex_lock_iothread(); + trace_postcopy_start_set_run(); + + qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL); + global_state_store(); + ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); + if (ret < 0) { + goto fail; + } + + ret = migration_maybe_pause(ms, &cur_state, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + if (ret < 0) { + goto fail; + } + + ret = bdrv_inactivate_all(); + if (ret < 0) { + goto fail; + } + restart_block = true; + + /* + * Cause any non-postcopiable, but iterative devices to + * send out their final data. + */ + qemu_savevm_state_complete_precopy(ms->to_dst_file, true, false); + + /* + * in Finish migrate and with the io-lock held everything should + * be quiet, but we've potentially still got dirty pages and we + * need to tell the destination to throw any pages it's already received + * that are dirty + */ + if (migrate_postcopy_ram()) { + if (ram_postcopy_send_discard_bitmap(ms)) { + error_report("postcopy send discard bitmap failed"); + goto fail; + } + } + + /* + * send rest of state - note things that are doing postcopy + * will notice we're in POSTCOPY_ACTIVE and not actually + * wrap their state up here + */ + /* 0 max-postcopy-bandwidth means unlimited */ + if (!bandwidth) { + qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX); + } else { + qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO); + } + if (migrate_postcopy_ram()) { + /* Ping just for debugging, helps line traces up */ + qemu_savevm_send_ping(ms->to_dst_file, 2); + } + + /* + * While loading the device state we may trigger page transfer + * requests and the fd must be free to process those, and thus + * the destination must read the whole device state off the fd before + * it starts processing it. Unfortunately the ad-hoc migration format + * doesn't allow the destination to know the size to read without fully + * parsing it through each devices load-state code (especially the open + * coded devices that use get/put). + * So we wrap the device state up in a package with a length at the start; + * to do this we use a qemu_buf to hold the whole of the device state. + */ + bioc = qio_channel_buffer_new(4096); + qio_channel_set_name(QIO_CHANNEL(bioc), "migration-postcopy-buffer"); + fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc)); + object_unref(OBJECT(bioc)); + + /* + * Make sure the receiver can get incoming pages before we send the rest + * of the state + */ + qemu_savevm_send_postcopy_listen(fb); + + qemu_savevm_state_complete_precopy(fb, false, false); + if (migrate_postcopy_ram()) { + qemu_savevm_send_ping(fb, 3); + } + + qemu_savevm_send_postcopy_run(fb); + + /* <><> end of stuff going into the package */ + + /* Last point of recovery; as soon as we send the package the destination + * can open devices and potentially start running. + * Lets just check again we've not got any errors. + */ + ret = qemu_file_get_error(ms->to_dst_file); + if (ret) { + error_report("postcopy_start: Migration stream errored (pre package)"); + goto fail_closefb; + } + + restart_block = false; + + /* Now send that blob */ + if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) { + goto fail_closefb; + } + qemu_fclose(fb); + + /* Send a notify to give a chance for anything that needs to happen + * at the transition to postcopy and after the device state; in particular + * spice needs to trigger a transition now + */ + ms->postcopy_after_devices = true; + notifier_list_notify(&migration_state_notifiers, ms); + + ms->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop; + + qemu_mutex_unlock_iothread(); + + if (migrate_postcopy_ram()) { + /* + * Although this ping is just for debug, it could potentially be + * used for getting a better measurement of downtime at the source. + */ + qemu_savevm_send_ping(ms->to_dst_file, 4); + } + + if (migrate_release_ram()) { + ram_postcopy_migrated_memory_release(ms); + } + + ret = qemu_file_get_error(ms->to_dst_file); + if (ret) { + error_report("postcopy_start: Migration stream errored"); + migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_FAILED); + } + + return ret; + +fail_closefb: + qemu_fclose(fb); +fail: + migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_FAILED); + if (restart_block) { + /* A failure happened early enough that we know the destination hasn't + * accessed block devices, so we're safe to recover. + */ + Error *local_err = NULL; + + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + } + } + qemu_mutex_unlock_iothread(); + return -1; +} + +/** + * migration_maybe_pause: Pause if required to by + * migrate_pause_before_switchover called with the iothread locked + * Returns: 0 on success + */ +static int migration_maybe_pause(MigrationState *s, + int *current_active_state, + int new_state) +{ + if (!migrate_pause_before_switchover()) { + return 0; + } + + /* Since leaving this state is not atomic with posting the semaphore + * it's possible that someone could have issued multiple migrate_continue + * and the semaphore is incorrectly positive at this point; + * the docs say it's undefined to reinit a semaphore that's already + * init'd, so use timedwait to eat up any existing posts. + */ + while (qemu_sem_timedwait(&s->pause_sem, 1) == 0) { + /* This block intentionally left blank */ + } + + /* + * If the migration is cancelled when it is in the completion phase, + * the migration state is set to MIGRATION_STATUS_CANCELLING. + * So we don't need to wait a semaphore, otherwise we would always + * wait for the 'pause_sem' semaphore. + */ + if (s->state != MIGRATION_STATUS_CANCELLING) { + qemu_mutex_unlock_iothread(); + migrate_set_state(&s->state, *current_active_state, + MIGRATION_STATUS_PRE_SWITCHOVER); + qemu_sem_wait(&s->pause_sem); + migrate_set_state(&s->state, MIGRATION_STATUS_PRE_SWITCHOVER, + new_state); + *current_active_state = new_state; + qemu_mutex_lock_iothread(); + } + + return s->state == new_state ? 0 : -EINVAL; +} + +/** + * migration_completion: Used by migration_thread when there's not much left. + * The caller 'breaks' the loop when this returns. + * + * @s: Current migration state + */ +static void migration_completion(MigrationState *s) +{ + int ret; + int current_active_state = s->state; + + if (s->state == MIGRATION_STATUS_ACTIVE) { + qemu_mutex_lock_iothread(); + s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL); + s->vm_was_running = runstate_is_running(); + ret = global_state_store(); + + if (!ret) { + bool inactivate = !migrate_colo_enabled(); + ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); + trace_migration_completion_vm_stop(ret); + if (ret >= 0) { + ret = migration_maybe_pause(s, ¤t_active_state, + MIGRATION_STATUS_DEVICE); + } + if (ret >= 0) { + qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX); + ret = qemu_savevm_state_complete_precopy(s->to_dst_file, false, + inactivate); + } + if (inactivate && ret >= 0) { + s->block_inactive = true; + } + } + qemu_mutex_unlock_iothread(); + + if (ret < 0) { + goto fail; + } + } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) { + trace_migration_completion_postcopy_end(); + + qemu_mutex_lock_iothread(); + qemu_savevm_state_complete_postcopy(s->to_dst_file); + qemu_mutex_unlock_iothread(); + + trace_migration_completion_postcopy_end_after_complete(); + } else if (s->state == MIGRATION_STATUS_CANCELLING) { + goto fail; + } + + /* + * If rp was opened we must clean up the thread before + * cleaning everything else up (since if there are no failures + * it will wait for the destination to send it's status in + * a SHUT command). + */ + if (s->rp_state.rp_thread_created) { + int rp_error; + trace_migration_return_path_end_before(); + rp_error = await_return_path_close_on_source(s); + trace_migration_return_path_end_after(rp_error); + if (rp_error) { + goto fail_invalidate; + } + } + + if (qemu_file_get_error(s->to_dst_file)) { + trace_migration_completion_file_err(); + goto fail_invalidate; + } + + if (!migrate_colo_enabled()) { + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_COMPLETED); + } + + return; + +fail_invalidate: + /* If not doing postcopy, vm_start() will be called: let's regain + * control on images. + */ + if (s->state == MIGRATION_STATUS_ACTIVE || + s->state == MIGRATION_STATUS_DEVICE) { + Error *local_err = NULL; + + qemu_mutex_lock_iothread(); + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + } else { + s->block_inactive = false; + } + qemu_mutex_unlock_iothread(); + } + +fail: + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_FAILED); +} + +/** + * bg_migration_completion: Used by bg_migration_thread when after all the + * RAM has been saved. The caller 'breaks' the loop when this returns. + * + * @s: Current migration state + */ +static void bg_migration_completion(MigrationState *s) +{ + int current_active_state = s->state; + + /* + * Stop tracking RAM writes - un-protect memory, un-register UFFD + * memory ranges, flush kernel wait queues and wake up threads + * waiting for write fault to be resolved. + */ + ram_write_tracking_stop(); + + if (s->state == MIGRATION_STATUS_ACTIVE) { + /* + * By this moment we have RAM content saved into the migration stream. + * The next step is to flush the non-RAM content (device state) + * right after the ram content. The device state has been stored into + * the temporary buffer before RAM saving started. + */ + qemu_put_buffer(s->to_dst_file, s->bioc->data, s->bioc->usage); + qemu_fflush(s->to_dst_file); + } else if (s->state == MIGRATION_STATUS_CANCELLING) { + goto fail; + } + + if (qemu_file_get_error(s->to_dst_file)) { + trace_migration_completion_file_err(); + goto fail; + } + + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_COMPLETED); + return; + +fail: + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_FAILED); +} + +bool migrate_colo_enabled(void) +{ + MigrationState *s = migrate_get_current(); + return s->enabled_capabilities[MIGRATION_CAPABILITY_X_COLO]; +} + +typedef enum MigThrError { + /* No error detected */ + MIG_THR_ERR_NONE = 0, + /* Detected error, but resumed successfully */ + MIG_THR_ERR_RECOVERED = 1, + /* Detected fatal error, need to exit */ + MIG_THR_ERR_FATAL = 2, +} MigThrError; + +static int postcopy_resume_handshake(MigrationState *s) +{ + qemu_savevm_send_postcopy_resume(s->to_dst_file); + + while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) { + qemu_sem_wait(&s->rp_state.rp_sem); + } + + if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) { + return 0; + } + + return -1; +} + +/* Return zero if success, or <0 for error */ +static int postcopy_do_resume(MigrationState *s) +{ + int ret; + + /* + * Call all the resume_prepare() hooks, so that modules can be + * ready for the migration resume. + */ + ret = qemu_savevm_state_resume_prepare(s); + if (ret) { + error_report("%s: resume_prepare() failure detected: %d", + __func__, ret); + return ret; + } + + /* + * Last handshake with destination on the resume (destination will + * switch to postcopy-active afterwards) + */ + ret = postcopy_resume_handshake(s); + if (ret) { + error_report("%s: handshake failed: %d", __func__, ret); + return ret; + } + + return 0; +} + +/* + * We don't return until we are in a safe state to continue current + * postcopy migration. Returns MIG_THR_ERR_RECOVERED if recovered, or + * MIG_THR_ERR_FATAL if unrecovery failure happened. + */ +static MigThrError postcopy_pause(MigrationState *s) +{ + assert(s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE); + + while (true) { + QEMUFile *file; + + /* + * Current channel is possibly broken. Release it. Note that this is + * guaranteed even without lock because to_dst_file should only be + * modified by the migration thread. That also guarantees that the + * unregister of yank is safe too without the lock. It should be safe + * even to be within the qemu_file_lock, but we didn't do that to avoid + * taking more mutex (yank_lock) within qemu_file_lock. TL;DR: we make + * the qemu_file_lock critical section as small as possible. + */ + assert(s->to_dst_file); + migration_ioc_unregister_yank_from_file(s->to_dst_file); + qemu_mutex_lock(&s->qemu_file_lock); + file = s->to_dst_file; + s->to_dst_file = NULL; + qemu_mutex_unlock(&s->qemu_file_lock); + + qemu_file_shutdown(file); + qemu_fclose(file); + + migrate_set_state(&s->state, s->state, + MIGRATION_STATUS_POSTCOPY_PAUSED); + + error_report("Detected IO failure for postcopy. " + "Migration paused."); + + /* + * We wait until things fixed up. Then someone will setup the + * status back for us. + */ + while (s->state == MIGRATION_STATUS_POSTCOPY_PAUSED) { + qemu_sem_wait(&s->postcopy_pause_sem); + } + + if (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) { + /* Woken up by a recover procedure. Give it a shot */ + + /* + * Firstly, let's wake up the return path now, with a new + * return path channel. + */ + qemu_sem_post(&s->postcopy_pause_rp_sem); + + /* Do the resume logic */ + if (postcopy_do_resume(s) == 0) { + /* Let's continue! */ + trace_postcopy_pause_continued(); + return MIG_THR_ERR_RECOVERED; + } else { + /* + * Something wrong happened during the recovery, let's + * pause again. Pause is always better than throwing + * data away. + */ + continue; + } + } else { + /* This is not right... Time to quit. */ + return MIG_THR_ERR_FATAL; + } + } +} + +static MigThrError migration_detect_error(MigrationState *s) +{ + int ret; + int state = s->state; + Error *local_error = NULL; + + if (state == MIGRATION_STATUS_CANCELLING || + state == MIGRATION_STATUS_CANCELLED) { + /* End the migration, but don't set the state to failed */ + return MIG_THR_ERR_FATAL; + } + + /* Try to detect any file errors */ + ret = qemu_file_get_error_obj(s->to_dst_file, &local_error); + if (!ret) { + /* Everything is fine */ + assert(!local_error); + return MIG_THR_ERR_NONE; + } + + if (local_error) { + migrate_set_error(s, local_error); + error_free(local_error); + } + + if (state == MIGRATION_STATUS_POSTCOPY_ACTIVE && ret == -EIO) { + /* + * For postcopy, we allow the network to be down for a + * while. After that, it can be continued by a + * recovery phase. + */ + return postcopy_pause(s); + } else { + /* + * For precopy (or postcopy with error outside IO), we fail + * with no time. + */ + migrate_set_state(&s->state, state, MIGRATION_STATUS_FAILED); + trace_migration_thread_file_err(); + + /* Time to stop the migration, now. */ + return MIG_THR_ERR_FATAL; + } +} + +/* How many bytes have we transferred since the beginning of the migration */ +static uint64_t migration_total_bytes(MigrationState *s) +{ + return qemu_ftell(s->to_dst_file) + ram_counters.multifd_bytes; +} + +static void migration_calculate_complete(MigrationState *s) +{ + uint64_t bytes = migration_total_bytes(s); + int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + int64_t transfer_time; + + s->total_time = end_time - s->start_time; + if (!s->downtime) { + /* + * It's still not set, so we are precopy migration. For + * postcopy, downtime is calculated during postcopy_start(). + */ + s->downtime = end_time - s->downtime_start; + } + + transfer_time = s->total_time - s->setup_time; + if (transfer_time) { + s->mbps = ((double) bytes * 8.0) / transfer_time / 1000; + } +} + +static void update_iteration_initial_status(MigrationState *s) +{ + /* + * Update these three fields at the same time to avoid mismatch info lead + * wrong speed calculation. + */ + s->iteration_start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + s->iteration_initial_bytes = migration_total_bytes(s); + s->iteration_initial_pages = ram_get_total_transferred_pages(); +} + +static void migration_update_counters(MigrationState *s, + int64_t current_time) +{ + uint64_t transferred, transferred_pages, time_spent; + uint64_t current_bytes; /* bytes transferred since the beginning */ + double bandwidth; + + if (current_time < s->iteration_start_time + BUFFER_DELAY) { + return; + } + + current_bytes = migration_total_bytes(s); + transferred = current_bytes - s->iteration_initial_bytes; + time_spent = current_time - s->iteration_start_time; + bandwidth = (double)transferred / time_spent; + s->threshold_size = bandwidth * s->parameters.downtime_limit; + + s->mbps = (((double) transferred * 8.0) / + ((double) time_spent / 1000.0)) / 1000.0 / 1000.0; + + transferred_pages = ram_get_total_transferred_pages() - + s->iteration_initial_pages; + s->pages_per_second = (double) transferred_pages / + (((double) time_spent / 1000.0)); + + /* + * if we haven't sent anything, we don't want to + * recalculate. 10000 is a small enough number for our purposes + */ + if (ram_counters.dirty_pages_rate && transferred > 10000) { + s->expected_downtime = ram_counters.remaining / bandwidth; + } + + qemu_file_reset_rate_limit(s->to_dst_file); + + update_iteration_initial_status(s); + + trace_migrate_transferred(transferred, time_spent, + bandwidth, s->threshold_size); +} + +/* Migration thread iteration status */ +typedef enum { + MIG_ITERATE_RESUME, /* Resume current iteration */ + MIG_ITERATE_SKIP, /* Skip current iteration */ + MIG_ITERATE_BREAK, /* Break the loop */ +} MigIterateState; + +/* + * Return true if continue to the next iteration directly, false + * otherwise. + */ +static MigIterateState migration_iteration_run(MigrationState *s) +{ + uint64_t pending_size, pend_pre, pend_compat, pend_post; + bool in_postcopy = s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE; + + qemu_savevm_state_pending(s->to_dst_file, s->threshold_size, &pend_pre, + &pend_compat, &pend_post); + pending_size = pend_pre + pend_compat + pend_post; + + trace_migrate_pending(pending_size, s->threshold_size, + pend_pre, pend_compat, pend_post); + + if (pending_size && pending_size >= s->threshold_size) { + /* Still a significant amount to transfer */ + if (!in_postcopy && pend_pre <= s->threshold_size && + qatomic_read(&s->start_postcopy)) { + if (postcopy_start(s)) { + error_report("%s: postcopy failed to start", __func__); + } + return MIG_ITERATE_SKIP; + } + /* Just another iteration step */ + qemu_savevm_state_iterate(s->to_dst_file, in_postcopy); + } else { + trace_migration_thread_low_pending(pending_size); + migration_completion(s); + return MIG_ITERATE_BREAK; + } + + return MIG_ITERATE_RESUME; +} + +static void migration_iteration_finish(MigrationState *s) +{ + /* If we enabled cpu throttling for auto-converge, turn it off. */ + cpu_throttle_stop(); + + qemu_mutex_lock_iothread(); + switch (s->state) { + case MIGRATION_STATUS_COMPLETED: + migration_calculate_complete(s); + runstate_set(RUN_STATE_POSTMIGRATE); + break; + + case MIGRATION_STATUS_ACTIVE: + /* + * We should really assert here, but since it's during + * migration, let's try to reduce the usage of assertions. + */ + if (!migrate_colo_enabled()) { + error_report("%s: critical error: calling COLO code without " + "COLO enabled", __func__); + } + migrate_start_colo_process(s); + /* + * Fixme: we will run VM in COLO no matter its old running state. + * After exited COLO, we will keep running. + */ + s->vm_was_running = true; + /* Fallthrough */ + case MIGRATION_STATUS_FAILED: + case MIGRATION_STATUS_CANCELLED: + case MIGRATION_STATUS_CANCELLING: + if (s->vm_was_running) { + if (!runstate_check(RUN_STATE_SHUTDOWN)) { + vm_start(); + } + } else { + if (runstate_check(RUN_STATE_FINISH_MIGRATE)) { + runstate_set(RUN_STATE_POSTMIGRATE); + } + } + break; + + default: + /* Should not reach here, but if so, forgive the VM. */ + error_report("%s: Unknown ending state %d", __func__, s->state); + break; + } + migrate_fd_cleanup_schedule(s); + qemu_mutex_unlock_iothread(); +} + +static void bg_migration_iteration_finish(MigrationState *s) +{ + qemu_mutex_lock_iothread(); + switch (s->state) { + case MIGRATION_STATUS_COMPLETED: + migration_calculate_complete(s); + break; + + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_FAILED: + case MIGRATION_STATUS_CANCELLED: + case MIGRATION_STATUS_CANCELLING: + break; + + default: + /* Should not reach here, but if so, forgive the VM. */ + error_report("%s: Unknown ending state %d", __func__, s->state); + break; + } + + migrate_fd_cleanup_schedule(s); + qemu_mutex_unlock_iothread(); +} + +/* + * Return true if continue to the next iteration directly, false + * otherwise. + */ +static MigIterateState bg_migration_iteration_run(MigrationState *s) +{ + int res; + + res = qemu_savevm_state_iterate(s->to_dst_file, false); + if (res > 0) { + bg_migration_completion(s); + return MIG_ITERATE_BREAK; + } + + return MIG_ITERATE_RESUME; +} + +void migration_make_urgent_request(void) +{ + qemu_sem_post(&migrate_get_current()->rate_limit_sem); +} + +void migration_consume_urgent_request(void) +{ + qemu_sem_wait(&migrate_get_current()->rate_limit_sem); +} + +/* Returns true if the rate limiting was broken by an urgent request */ +bool migration_rate_limit(void) +{ + int64_t now = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + MigrationState *s = migrate_get_current(); + + bool urgent = false; + migration_update_counters(s, now); + if (qemu_file_rate_limit(s->to_dst_file)) { + + if (qemu_file_get_error(s->to_dst_file)) { + return false; + } + /* + * Wait for a delay to do rate limiting OR + * something urgent to post the semaphore. + */ + int ms = s->iteration_start_time + BUFFER_DELAY - now; + trace_migration_rate_limit_pre(ms); + if (qemu_sem_timedwait(&s->rate_limit_sem, ms) == 0) { + /* + * We were woken by one or more urgent things but + * the timedwait will have consumed one of them. + * The service routine for the urgent wake will dec + * the semaphore itself for each item it consumes, + * so add this one we just eat back. + */ + qemu_sem_post(&s->rate_limit_sem); + urgent = true; + } + trace_migration_rate_limit_post(urgent); + } + return urgent; +} + +/* + * if failover devices are present, wait they are completely + * unplugged + */ + +static void qemu_savevm_wait_unplug(MigrationState *s, int old_state, + int new_state) +{ + if (qemu_savevm_state_guest_unplug_pending()) { + migrate_set_state(&s->state, old_state, MIGRATION_STATUS_WAIT_UNPLUG); + + while (s->state == MIGRATION_STATUS_WAIT_UNPLUG && + qemu_savevm_state_guest_unplug_pending()) { + qemu_sem_timedwait(&s->wait_unplug_sem, 250); + } + if (s->state != MIGRATION_STATUS_WAIT_UNPLUG) { + int timeout = 120; /* 30 seconds */ + /* + * migration has been canceled + * but as we have started an unplug we must wait the end + * to be able to plug back the card + */ + while (timeout-- && qemu_savevm_state_guest_unplug_pending()) { + qemu_sem_timedwait(&s->wait_unplug_sem, 250); + } + if (qemu_savevm_state_guest_unplug_pending()) { + warn_report("migration: partially unplugged device on " + "failure"); + } + } + + migrate_set_state(&s->state, MIGRATION_STATUS_WAIT_UNPLUG, new_state); + } else { + migrate_set_state(&s->state, old_state, new_state); + } +} + +/* + * Master migration thread on the source VM. + * It drives the migration and pumps the data down the outgoing channel. + */ +static void *migration_thread(void *opaque) +{ + MigrationState *s = opaque; + int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); + MigThrError thr_error; + bool urgent = false; + + rcu_register_thread(); + + object_ref(OBJECT(s)); + update_iteration_initial_status(s); + + qemu_savevm_state_header(s->to_dst_file); + + /* + * If we opened the return path, we need to make sure dst has it + * opened as well. + */ + if (s->rp_state.rp_thread_created) { + /* Now tell the dest that it should open its end so it can reply */ + qemu_savevm_send_open_return_path(s->to_dst_file); + + /* And do a ping that will make stuff easier to debug */ + qemu_savevm_send_ping(s->to_dst_file, 1); + } + + if (migrate_postcopy()) { + /* + * Tell the destination that we *might* want to do postcopy later; + * if the other end can't do postcopy it should fail now, nice and + * early. + */ + qemu_savevm_send_postcopy_advise(s->to_dst_file); + } + + if (migrate_colo_enabled()) { + /* Notify migration destination that we enable COLO */ + qemu_savevm_send_colo_enable(s->to_dst_file); + } + + qemu_savevm_state_setup(s->to_dst_file); + + qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_ACTIVE); + + s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; + + trace_migration_thread_setup_complete(); + + while (migration_is_active(s)) { + if (urgent || !qemu_file_rate_limit(s->to_dst_file)) { + MigIterateState iter_state = migration_iteration_run(s); + if (iter_state == MIG_ITERATE_SKIP) { + continue; + } else if (iter_state == MIG_ITERATE_BREAK) { + break; + } + } + + /* + * Try to detect any kind of failures, and see whether we + * should stop the migration now. + */ + thr_error = migration_detect_error(s); + if (thr_error == MIG_THR_ERR_FATAL) { + /* Stop migration */ + break; + } else if (thr_error == MIG_THR_ERR_RECOVERED) { + /* + * Just recovered from a e.g. network failure, reset all + * the local variables. This is important to avoid + * breaking transferred_bytes and bandwidth calculation + */ + update_iteration_initial_status(s); + } + + urgent = migration_rate_limit(); + } + + trace_migration_thread_after_loop(); + migration_iteration_finish(s); + object_unref(OBJECT(s)); + rcu_unregister_thread(); + return NULL; +} + +static void bg_migration_vm_start_bh(void *opaque) +{ + MigrationState *s = opaque; + + qemu_bh_delete(s->vm_start_bh); + s->vm_start_bh = NULL; + + vm_start(); + s->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - s->downtime_start; +} + +/** + * Background snapshot thread, based on live migration code. + * This is an alternative implementation of live migration mechanism + * introduced specifically to support background snapshots. + * + * It takes advantage of userfault_fd write protection mechanism introduced + * in v5.7 kernel. Compared to existing dirty page logging migration much + * lesser stream traffic is produced resulting in smaller snapshot images, + * simply cause of no page duplicates can get into the stream. + * + * Another key point is that generated vmstate stream reflects machine state + * 'frozen' at the beginning of snapshot creation compared to dirty page logging + * mechanism, which effectively results in that saved snapshot is the state of VM + * at the end of the process. + */ +static void *bg_migration_thread(void *opaque) +{ + MigrationState *s = opaque; + int64_t setup_start; + MigThrError thr_error; + QEMUFile *fb; + bool early_fail = true; + + rcu_register_thread(); + object_ref(OBJECT(s)); + + qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX); + + setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); + /* + * We want to save vmstate for the moment when migration has been + * initiated but also we want to save RAM content while VM is running. + * The RAM content should appear first in the vmstate. So, we first + * stash the non-RAM part of the vmstate to the temporary buffer, + * then write RAM part of the vmstate to the migration stream + * with vCPUs running and, finally, write stashed non-RAM part of + * the vmstate from the buffer to the migration stream. + */ + s->bioc = qio_channel_buffer_new(512 * 1024); + qio_channel_set_name(QIO_CHANNEL(s->bioc), "vmstate-buffer"); + fb = qemu_fopen_channel_output(QIO_CHANNEL(s->bioc)); + object_unref(OBJECT(s->bioc)); + + update_iteration_initial_status(s); + + /* + * Prepare for tracking memory writes with UFFD-WP - populate + * RAM pages before protecting. + */ +#ifdef __linux__ + ram_write_tracking_prepare(); +#endif + + qemu_savevm_state_header(s->to_dst_file); + qemu_savevm_state_setup(s->to_dst_file); + + qemu_savevm_wait_unplug(s, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_ACTIVE); + + s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; + + trace_migration_thread_setup_complete(); + s->downtime_start = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + + qemu_mutex_lock_iothread(); + + /* + * If VM is currently in suspended state, then, to make a valid runstate + * transition in vm_stop_force_state() we need to wakeup it up. + */ + qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER, NULL); + s->vm_was_running = runstate_is_running(); + + if (global_state_store()) { + goto fail; + } + /* Forcibly stop VM before saving state of vCPUs and devices */ + if (vm_stop_force_state(RUN_STATE_PAUSED)) { + goto fail; + } + /* + * Put vCPUs in sync with shadow context structures, then + * save their state to channel-buffer along with devices. + */ + cpu_synchronize_all_states(); + if (qemu_savevm_state_complete_precopy_non_iterable(fb, false, false)) { + goto fail; + } + /* + * Since we are going to get non-iterable state data directly + * from s->bioc->data, explicit flush is needed here. + */ + qemu_fflush(fb); + + /* Now initialize UFFD context and start tracking RAM writes */ + if (ram_write_tracking_start()) { + goto fail; + } + early_fail = false; + + /* + * Start VM from BH handler to avoid write-fault lock here. + * UFFD-WP protection for the whole RAM is already enabled so + * calling VM state change notifiers from vm_start() would initiate + * writes to virtio VQs memory which is in write-protected region. + */ + s->vm_start_bh = qemu_bh_new(bg_migration_vm_start_bh, s); + qemu_bh_schedule(s->vm_start_bh); + + qemu_mutex_unlock_iothread(); + + while (migration_is_active(s)) { + MigIterateState iter_state = bg_migration_iteration_run(s); + if (iter_state == MIG_ITERATE_SKIP) { + continue; + } else if (iter_state == MIG_ITERATE_BREAK) { + break; + } + + /* + * Try to detect any kind of failures, and see whether we + * should stop the migration now. + */ + thr_error = migration_detect_error(s); + if (thr_error == MIG_THR_ERR_FATAL) { + /* Stop migration */ + break; + } + + migration_update_counters(s, qemu_clock_get_ms(QEMU_CLOCK_REALTIME)); + } + + trace_migration_thread_after_loop(); + +fail: + if (early_fail) { + migrate_set_state(&s->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_FAILED); + qemu_mutex_unlock_iothread(); + } + + bg_migration_iteration_finish(s); + + qemu_fclose(fb); + object_unref(OBJECT(s)); + rcu_unregister_thread(); + + return NULL; +} + +void migrate_fd_connect(MigrationState *s, Error *error_in) +{ + Error *local_err = NULL; + int64_t rate_limit; + bool resume = s->state == MIGRATION_STATUS_POSTCOPY_PAUSED; + + /* + * If there's a previous error, free it and prepare for another one. + * Meanwhile if migration completes successfully, there won't have an error + * dumped when calling migrate_fd_cleanup(). + */ + migrate_error_free(s); + + s->expected_downtime = s->parameters.downtime_limit; + if (resume) { + assert(s->cleanup_bh); + } else { + assert(!s->cleanup_bh); + s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup_bh, s); + } + if (error_in) { + migrate_fd_error(s, error_in); + if (resume) { + /* + * Don't do cleanup for resume if channel is invalid, but only dump + * the error. We wait for another channel connect from the user. + * The error_report still gives HMP user a hint on what failed. + * It's normally done in migrate_fd_cleanup(), but call it here + * explicitly. + */ + error_report_err(error_copy(s->error)); + } else { + migrate_fd_cleanup(s); + } + return; + } + + if (resume) { + /* This is a resumed migration */ + rate_limit = s->parameters.max_postcopy_bandwidth / + XFER_LIMIT_RATIO; + } else { + /* This is a fresh new migration */ + rate_limit = s->parameters.max_bandwidth / XFER_LIMIT_RATIO; + + /* Notify before starting migration thread */ + notifier_list_notify(&migration_state_notifiers, s); + } + + qemu_file_set_rate_limit(s->to_dst_file, rate_limit); + qemu_file_set_blocking(s->to_dst_file, true); + + /* + * Open the return path. For postcopy, it is used exclusively. For + * precopy, only if user specified "return-path" capability would + * QEMU uses the return path. + */ + if (migrate_postcopy_ram() || migrate_use_return_path()) { + if (open_return_path_on_source(s, !resume)) { + error_report("Unable to open return-path for postcopy"); + migrate_set_state(&s->state, s->state, MIGRATION_STATUS_FAILED); + migrate_fd_cleanup(s); + return; + } + } + + if (resume) { + /* Wakeup the main migration thread to do the recovery */ + migrate_set_state(&s->state, MIGRATION_STATUS_POSTCOPY_PAUSED, + MIGRATION_STATUS_POSTCOPY_RECOVER); + qemu_sem_post(&s->postcopy_pause_sem); + return; + } + + if (multifd_save_setup(&local_err) != 0) { + error_report_err(local_err); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); + migrate_fd_cleanup(s); + return; + } + + if (migrate_background_snapshot()) { + qemu_thread_create(&s->thread, "bg_snapshot", + bg_migration_thread, s, QEMU_THREAD_JOINABLE); + } else { + qemu_thread_create(&s->thread, "live_migration", + migration_thread, s, QEMU_THREAD_JOINABLE); + } + s->migration_thread_running = true; +} + +void migration_global_dump(Monitor *mon) +{ + MigrationState *ms = migrate_get_current(); + + monitor_printf(mon, "globals:\n"); + monitor_printf(mon, "store-global-state: %s\n", + ms->store_global_state ? "on" : "off"); + monitor_printf(mon, "only-migratable: %s\n", + only_migratable ? "on" : "off"); + monitor_printf(mon, "send-configuration: %s\n", + ms->send_configuration ? "on" : "off"); + monitor_printf(mon, "send-section-footer: %s\n", + ms->send_section_footer ? "on" : "off"); + monitor_printf(mon, "decompress-error-check: %s\n", + ms->decompress_error_check ? "on" : "off"); + monitor_printf(mon, "clear-bitmap-shift: %u\n", + ms->clear_bitmap_shift); +} + +#define DEFINE_PROP_MIG_CAP(name, x) \ + DEFINE_PROP_BOOL(name, MigrationState, enabled_capabilities[x], false) + +static Property migration_properties[] = { + DEFINE_PROP_BOOL("store-global-state", MigrationState, + store_global_state, true), + DEFINE_PROP_BOOL("send-configuration", MigrationState, + send_configuration, true), + DEFINE_PROP_BOOL("send-section-footer", MigrationState, + send_section_footer, true), + DEFINE_PROP_BOOL("decompress-error-check", MigrationState, + decompress_error_check, true), + DEFINE_PROP_UINT8("x-clear-bitmap-shift", MigrationState, + clear_bitmap_shift, CLEAR_BITMAP_SHIFT_DEFAULT), + + /* Migration parameters */ + DEFINE_PROP_UINT8("x-compress-level", MigrationState, + parameters.compress_level, + DEFAULT_MIGRATE_COMPRESS_LEVEL), + DEFINE_PROP_UINT8("x-compress-threads", MigrationState, + parameters.compress_threads, + DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT), + DEFINE_PROP_BOOL("x-compress-wait-thread", MigrationState, + parameters.compress_wait_thread, true), + DEFINE_PROP_UINT8("x-decompress-threads", MigrationState, + parameters.decompress_threads, + DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT), + DEFINE_PROP_UINT8("x-throttle-trigger-threshold", MigrationState, + parameters.throttle_trigger_threshold, + DEFAULT_MIGRATE_THROTTLE_TRIGGER_THRESHOLD), + DEFINE_PROP_UINT8("x-cpu-throttle-initial", MigrationState, + parameters.cpu_throttle_initial, + DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL), + DEFINE_PROP_UINT8("x-cpu-throttle-increment", MigrationState, + parameters.cpu_throttle_increment, + DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT), + DEFINE_PROP_BOOL("x-cpu-throttle-tailslow", MigrationState, + parameters.cpu_throttle_tailslow, false), + DEFINE_PROP_SIZE("x-max-bandwidth", MigrationState, + parameters.max_bandwidth, MAX_THROTTLE), + DEFINE_PROP_UINT64("x-downtime-limit", MigrationState, + parameters.downtime_limit, + DEFAULT_MIGRATE_SET_DOWNTIME), + DEFINE_PROP_UINT32("x-checkpoint-delay", MigrationState, + parameters.x_checkpoint_delay, + DEFAULT_MIGRATE_X_CHECKPOINT_DELAY), + DEFINE_PROP_UINT8("multifd-channels", MigrationState, + parameters.multifd_channels, + DEFAULT_MIGRATE_MULTIFD_CHANNELS), + DEFINE_PROP_MULTIFD_COMPRESSION("multifd-compression", MigrationState, + parameters.multifd_compression, + DEFAULT_MIGRATE_MULTIFD_COMPRESSION), + DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState, + parameters.multifd_zlib_level, + DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL), + DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState, + parameters.multifd_zstd_level, + DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL), + DEFINE_PROP_SIZE("xbzrle-cache-size", MigrationState, + parameters.xbzrle_cache_size, + DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE), + DEFINE_PROP_SIZE("max-postcopy-bandwidth", MigrationState, + parameters.max_postcopy_bandwidth, + DEFAULT_MIGRATE_MAX_POSTCOPY_BANDWIDTH), + DEFINE_PROP_UINT8("max-cpu-throttle", MigrationState, + parameters.max_cpu_throttle, + DEFAULT_MIGRATE_MAX_CPU_THROTTLE), + DEFINE_PROP_SIZE("announce-initial", MigrationState, + parameters.announce_initial, + DEFAULT_MIGRATE_ANNOUNCE_INITIAL), + DEFINE_PROP_SIZE("announce-max", MigrationState, + parameters.announce_max, + DEFAULT_MIGRATE_ANNOUNCE_MAX), + DEFINE_PROP_SIZE("announce-rounds", MigrationState, + parameters.announce_rounds, + DEFAULT_MIGRATE_ANNOUNCE_ROUNDS), + DEFINE_PROP_SIZE("announce-step", MigrationState, + parameters.announce_step, + DEFAULT_MIGRATE_ANNOUNCE_STEP), + + /* Migration capabilities */ + DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE), + DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL), + DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE), + DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS), + DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS), + DEFINE_PROP_MIG_CAP("x-events", MIGRATION_CAPABILITY_EVENTS), + DEFINE_PROP_MIG_CAP("x-postcopy-ram", MIGRATION_CAPABILITY_POSTCOPY_RAM), + DEFINE_PROP_MIG_CAP("x-colo", MIGRATION_CAPABILITY_X_COLO), + DEFINE_PROP_MIG_CAP("x-release-ram", MIGRATION_CAPABILITY_RELEASE_RAM), + DEFINE_PROP_MIG_CAP("x-block", MIGRATION_CAPABILITY_BLOCK), + DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH), + DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD), + DEFINE_PROP_MIG_CAP("x-background-snapshot", + MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT), + + DEFINE_PROP_END_OF_LIST(), +}; + +static void migration_class_init(ObjectClass *klass, void *data) +{ + DeviceClass *dc = DEVICE_CLASS(klass); + + dc->user_creatable = false; + device_class_set_props(dc, migration_properties); +} + +static void migration_instance_finalize(Object *obj) +{ + MigrationState *ms = MIGRATION_OBJ(obj); + MigrationParameters *params = &ms->parameters; + + qemu_mutex_destroy(&ms->error_mutex); + qemu_mutex_destroy(&ms->qemu_file_lock); + g_free(params->tls_hostname); + g_free(params->tls_creds); + qemu_sem_destroy(&ms->wait_unplug_sem); + qemu_sem_destroy(&ms->rate_limit_sem); + qemu_sem_destroy(&ms->pause_sem); + qemu_sem_destroy(&ms->postcopy_pause_sem); + qemu_sem_destroy(&ms->postcopy_pause_rp_sem); + qemu_sem_destroy(&ms->rp_state.rp_sem); + error_free(ms->error); +} + +static void migration_instance_init(Object *obj) +{ + MigrationState *ms = MIGRATION_OBJ(obj); + MigrationParameters *params = &ms->parameters; + + ms->state = MIGRATION_STATUS_NONE; + ms->mbps = -1; + ms->pages_per_second = -1; + qemu_sem_init(&ms->pause_sem, 0); + qemu_mutex_init(&ms->error_mutex); + + params->tls_hostname = g_strdup(""); + params->tls_creds = g_strdup(""); + + /* Set has_* up only for parameter checks */ + params->has_compress_level = true; + params->has_compress_threads = true; + params->has_decompress_threads = true; + params->has_throttle_trigger_threshold = true; + params->has_cpu_throttle_initial = true; + params->has_cpu_throttle_increment = true; + params->has_cpu_throttle_tailslow = true; + params->has_max_bandwidth = true; + params->has_downtime_limit = true; + params->has_x_checkpoint_delay = true; + params->has_block_incremental = true; + params->has_multifd_channels = true; + params->has_multifd_compression = true; + params->has_multifd_zlib_level = true; + params->has_multifd_zstd_level = true; + params->has_xbzrle_cache_size = true; + params->has_max_postcopy_bandwidth = true; + params->has_max_cpu_throttle = true; + params->has_announce_initial = true; + params->has_announce_max = true; + params->has_announce_rounds = true; + params->has_announce_step = true; + + qemu_sem_init(&ms->postcopy_pause_sem, 0); + qemu_sem_init(&ms->postcopy_pause_rp_sem, 0); + qemu_sem_init(&ms->rp_state.rp_sem, 0); + qemu_sem_init(&ms->rate_limit_sem, 0); + qemu_sem_init(&ms->wait_unplug_sem, 0); + qemu_mutex_init(&ms->qemu_file_lock); +} + +/* + * Return true if check pass, false otherwise. Error will be put + * inside errp if provided. + */ +static bool migration_object_check(MigrationState *ms, Error **errp) +{ + MigrationCapabilityStatusList *head = NULL; + /* Assuming all off */ + bool cap_list[MIGRATION_CAPABILITY__MAX] = { 0 }, ret; + int i; + + if (!migrate_params_check(&ms->parameters, errp)) { + return false; + } + + for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { + if (ms->enabled_capabilities[i]) { + QAPI_LIST_PREPEND(head, migrate_cap_add(i, true)); + } + } + + ret = migrate_caps_check(cap_list, head, errp); + + /* It works with head == NULL */ + qapi_free_MigrationCapabilityStatusList(head); + + return ret; +} + +static const TypeInfo migration_type = { + .name = TYPE_MIGRATION, + /* + * NOTE: TYPE_MIGRATION is not really a device, as the object is + * not created using qdev_new(), it is not attached to the qdev + * device tree, and it is never realized. + * + * TODO: Make this TYPE_OBJECT once QOM provides something like + * TYPE_DEVICE's "-global" properties. + */ + .parent = TYPE_DEVICE, + .class_init = migration_class_init, + .class_size = sizeof(MigrationClass), + .instance_size = sizeof(MigrationState), + .instance_init = migration_instance_init, + .instance_finalize = migration_instance_finalize, +}; + +static void register_migration_types(void) +{ + type_register_static(&migration_type); +} + +type_init(register_migration_types); diff --git a/migration/migration.h b/migration/migration.h new file mode 100644 index 000000000..8130b703e --- /dev/null +++ b/migration/migration.h @@ -0,0 +1,395 @@ +/* + * QEMU live migration + * + * Copyright IBM, Corp. 2008 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#ifndef QEMU_MIGRATION_H +#define QEMU_MIGRATION_H + +#include "exec/cpu-common.h" +#include "hw/qdev-core.h" +#include "qapi/qapi-types-migration.h" +#include "qemu/thread.h" +#include "qemu/coroutine_int.h" +#include "io/channel.h" +#include "io/channel-buffer.h" +#include "net/announce.h" +#include "qom/object.h" + +struct PostcopyBlocktimeContext; + +#define MIGRATION_RESUME_ACK_VALUE (1) + +/* + * 1<<6=64 pages -> 256K chunk when page size is 4K. This gives us + * the benefit that all the chunks are 64 pages aligned then the + * bitmaps are always aligned to LONG. + */ +#define CLEAR_BITMAP_SHIFT_MIN 6 +/* + * 1<<18=256K pages -> 1G chunk when page size is 4K. This is the + * default value to use if no one specified. + */ +#define CLEAR_BITMAP_SHIFT_DEFAULT 18 +/* + * 1<<31=2G pages -> 8T chunk when page size is 4K. This should be + * big enough and make sure we won't overflow easily. + */ +#define CLEAR_BITMAP_SHIFT_MAX 31 + +/* State for the incoming migration */ +struct MigrationIncomingState { + QEMUFile *from_src_file; + + /* A hook to allow cleanup at the end of incoming migration */ + void *transport_data; + void (*transport_cleanup)(void *data); + + /* + * Free at the start of the main state load, set as the main thread finishes + * loading state. + */ + QemuEvent main_thread_load_event; + + /* For network announces */ + AnnounceTimer announce_timer; + + size_t largest_page_size; + bool have_fault_thread; + QemuThread fault_thread; + QemuSemaphore fault_thread_sem; + /* Set this when we want the fault thread to quit */ + bool fault_thread_quit; + + bool have_listen_thread; + QemuThread listen_thread; + QemuSemaphore listen_thread_sem; + + /* For the kernel to send us notifications */ + int userfault_fd; + /* To notify the fault_thread to wake, e.g., when need to quit */ + int userfault_event_fd; + QEMUFile *to_src_file; + QemuMutex rp_mutex; /* We send replies from multiple threads */ + /* RAMBlock of last request sent to source */ + RAMBlock *last_rb; + void *postcopy_tmp_page; + void *postcopy_tmp_zero_page; + /* PostCopyFD's for external userfaultfds & handlers of shared memory */ + GArray *postcopy_remote_fds; + + QEMUBH *bh; + + int state; + + bool have_colo_incoming_thread; + QemuThread colo_incoming_thread; + /* The coroutine we should enter (back) after failover */ + Coroutine *migration_incoming_co; + QemuSemaphore colo_incoming_sem; + + /* + * PostcopyBlocktimeContext to keep information for postcopy + * live migration, to calculate vCPU block time + * */ + struct PostcopyBlocktimeContext *blocktime_ctx; + + /* notify PAUSED postcopy incoming migrations to try to continue */ + bool postcopy_recover_triggered; + QemuSemaphore postcopy_pause_sem_dst; + QemuSemaphore postcopy_pause_sem_fault; + + /* List of listening socket addresses */ + SocketAddressList *socket_address_list; + + /* A tree of pages that we requested to the source VM */ + GTree *page_requested; + /* For debugging purpose only, but would be nice to keep */ + int page_requested_count; + /* + * The mutex helps to maintain the requested pages that we sent to the + * source, IOW, to guarantee coherent between the page_requests tree and + * the per-ramblock receivedmap. Note! This does not guarantee consistency + * of the real page copy procedures (using UFFDIO_[ZERO]COPY). E.g., even + * if one bit in receivedmap is cleared, UFFDIO_COPY could have happened + * for that page already. This is intended so that the mutex won't + * serialize and blocked by slow operations like UFFDIO_* ioctls. However + * this should be enough to make sure the page_requested tree always + * contains valid information. + */ + QemuMutex page_request_mutex; +}; + +MigrationIncomingState *migration_incoming_get_current(void); +void migration_incoming_state_destroy(void); +/* + * Functions to work with blocktime context + */ +void fill_destination_postcopy_migration_info(MigrationInfo *info); + +#define TYPE_MIGRATION "migration" + +typedef struct MigrationClass MigrationClass; +DECLARE_OBJ_CHECKERS(MigrationState, MigrationClass, + MIGRATION_OBJ, TYPE_MIGRATION) + +struct MigrationClass { + /*< private >*/ + DeviceClass parent_class; +}; + +struct MigrationState { + /*< private >*/ + DeviceState parent_obj; + + /*< public >*/ + QemuThread thread; + QEMUBH *vm_start_bh; + QEMUBH *cleanup_bh; + /* Protected by qemu_file_lock */ + QEMUFile *to_dst_file; + QIOChannelBuffer *bioc; + /* + * Protects to_dst_file/from_dst_file pointers. We need to make sure we + * won't yield or hang during the critical section, since this lock will be + * used in OOB command handler. + */ + QemuMutex qemu_file_lock; + + /* + * Used to allow urgent requests to override rate limiting. + */ + QemuSemaphore rate_limit_sem; + + /* pages already send at the beginning of current iteration */ + uint64_t iteration_initial_pages; + + /* pages transferred per second */ + double pages_per_second; + + /* bytes already send at the beginning of current iteration */ + uint64_t iteration_initial_bytes; + /* time at the start of current iteration */ + int64_t iteration_start_time; + /* + * The final stage happens when the remaining data is smaller than + * this threshold; it's calculated from the requested downtime and + * measured bandwidth + */ + int64_t threshold_size; + + /* params from 'migrate-set-parameters' */ + MigrationParameters parameters; + + int state; + + /* State related to return path */ + struct { + /* Protected by qemu_file_lock */ + QEMUFile *from_dst_file; + QemuThread rp_thread; + bool error; + /* + * We can also check non-zero of rp_thread, but there's no "official" + * way to do this, so this bool makes it slightly more elegant. + * Checking from_dst_file for this is racy because from_dst_file will + * be cleared in the rp_thread! + */ + bool rp_thread_created; + QemuSemaphore rp_sem; + } rp_state; + + double mbps; + /* Timestamp when recent migration starts (ms) */ + int64_t start_time; + /* Total time used by latest migration (ms) */ + int64_t total_time; + /* Timestamp when VM is down (ms) to migrate the last stuff */ + int64_t downtime_start; + int64_t downtime; + int64_t expected_downtime; + bool enabled_capabilities[MIGRATION_CAPABILITY__MAX]; + int64_t setup_time; + /* + * Whether guest was running when we enter the completion stage. + * If migration is interrupted by any reason, we need to continue + * running the guest on source. + */ + bool vm_was_running; + + /* Flag set once the migration has been asked to enter postcopy */ + bool start_postcopy; + /* Flag set after postcopy has sent the device state */ + bool postcopy_after_devices; + + /* Flag set once the migration thread is running (and needs joining) */ + bool migration_thread_running; + + /* Flag set once the migration thread called bdrv_inactivate_all */ + bool block_inactive; + + /* Migration is waiting for guest to unplug device */ + QemuSemaphore wait_unplug_sem; + + /* Migration is paused due to pause-before-switchover */ + QemuSemaphore pause_sem; + + /* The semaphore is used to notify COLO thread that failover is finished */ + QemuSemaphore colo_exit_sem; + + /* The event is used to notify COLO thread to do checkpoint */ + QemuEvent colo_checkpoint_event; + int64_t colo_checkpoint_time; + QEMUTimer *colo_delay_timer; + + /* The first error that has occurred. + We used the mutex to be able to return the 1st error message */ + Error *error; + /* mutex to protect errp */ + QemuMutex error_mutex; + + /* Do we have to clean up -b/-i from old migrate parameters */ + /* This feature is deprecated and will be removed */ + bool must_remove_block_options; + + /* + * Global switch on whether we need to store the global state + * during migration. + */ + bool store_global_state; + + /* Whether we send QEMU_VM_CONFIGURATION during migration */ + bool send_configuration; + /* Whether we send section footer during migration */ + bool send_section_footer; + + /* Needed by postcopy-pause state */ + QemuSemaphore postcopy_pause_sem; + QemuSemaphore postcopy_pause_rp_sem; + /* + * Whether we abort the migration if decompression errors are + * detected at the destination. It is left at false for qemu + * older than 3.0, since only newer qemu sends streams that + * do not trigger spurious decompression errors. + */ + bool decompress_error_check; + + /* + * This decides the size of guest memory chunk that will be used + * to track dirty bitmap clearing. The size of memory chunk will + * be GUEST_PAGE_SIZE << N. Say, N=0 means we will clear dirty + * bitmap for each page to send (1<<0=1); N=10 means we will clear + * dirty bitmap only once for 1<<10=1K continuous guest pages + * (which is in 4M chunk). + */ + uint8_t clear_bitmap_shift; + + /* + * This save hostname when out-going migration starts + */ + char *hostname; +}; + +void migrate_set_state(int *state, int old_state, int new_state); + +void migration_fd_process_incoming(QEMUFile *f, Error **errp); +void migration_ioc_process_incoming(QIOChannel *ioc, Error **errp); +void migration_incoming_process(void); + +bool migration_has_all_channels(void); + +uint64_t migrate_max_downtime(void); + +void migrate_set_error(MigrationState *s, const Error *error); +void migrate_fd_error(MigrationState *s, const Error *error); + +void migrate_fd_connect(MigrationState *s, Error *error_in); + +bool migration_is_setup_or_active(int state); +bool migration_is_running(int state); + +void migrate_init(MigrationState *s); +bool migration_is_blocked(Error **errp); +/* True if outgoing migration has entered postcopy phase */ +bool migration_in_postcopy(void); +MigrationState *migrate_get_current(void); + +bool migrate_postcopy(void); + +bool migrate_release_ram(void); +bool migrate_postcopy_ram(void); +bool migrate_zero_blocks(void); +bool migrate_dirty_bitmaps(void); +bool migrate_ignore_shared(void); +bool migrate_validate_uuid(void); + +bool migrate_auto_converge(void); +bool migrate_use_multifd(void); +bool migrate_pause_before_switchover(void); +int migrate_multifd_channels(void); +MultiFDCompression migrate_multifd_compression(void); +int migrate_multifd_zlib_level(void); +int migrate_multifd_zstd_level(void); + +int migrate_use_xbzrle(void); +uint64_t migrate_xbzrle_cache_size(void); +bool migrate_colo_enabled(void); + +bool migrate_use_block(void); +bool migrate_use_block_incremental(void); +int migrate_max_cpu_throttle(void); +bool migrate_use_return_path(void); + +uint64_t ram_get_total_transferred_pages(void); + +bool migrate_use_compression(void); +int migrate_compress_level(void); +int migrate_compress_threads(void); +int migrate_compress_wait_thread(void); +int migrate_decompress_threads(void); +bool migrate_use_events(void); +bool migrate_postcopy_blocktime(void); +bool migrate_background_snapshot(void); + +/* Sending on the return path - generic and then for each message type */ +void migrate_send_rp_shut(MigrationIncomingState *mis, + uint32_t value); +void migrate_send_rp_pong(MigrationIncomingState *mis, + uint32_t value); +int migrate_send_rp_req_pages(MigrationIncomingState *mis, RAMBlock *rb, + ram_addr_t start, uint64_t haddr); +int migrate_send_rp_message_req_pages(MigrationIncomingState *mis, + RAMBlock *rb, ram_addr_t start); +void migrate_send_rp_recv_bitmap(MigrationIncomingState *mis, + char *block_name); +void migrate_send_rp_resume_ack(MigrationIncomingState *mis, uint32_t value); + +void dirty_bitmap_mig_before_vm_start(void); +void dirty_bitmap_mig_cancel_outgoing(void); +void dirty_bitmap_mig_cancel_incoming(void); +bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm, + Error **errp); + +void migrate_add_address(SocketAddress *address); + +int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque); + +#define qemu_ram_foreach_block \ + #warning "Use foreach_not_ignored_block in migration code" + +void migration_make_urgent_request(void); +void migration_consume_urgent_request(void); +bool migration_rate_limit(void); +void migration_cancel(const Error *error); + +void populate_vfio_info(MigrationInfo *info); + +#endif diff --git a/migration/multifd-zlib.c b/migration/multifd-zlib.c new file mode 100644 index 000000000..ab4ba75d7 --- /dev/null +++ b/migration/multifd-zlib.c @@ -0,0 +1,325 @@ +/* + * Multifd zlib compression implementation + * + * Copyright (c) 2020 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include <zlib.h> +#include "qemu/rcu.h" +#include "exec/target_page.h" +#include "qapi/error.h" +#include "migration.h" +#include "trace.h" +#include "multifd.h" + +struct zlib_data { + /* stream for compression */ + z_stream zs; + /* compressed buffer */ + uint8_t *zbuff; + /* size of compressed buffer */ + uint32_t zbuff_len; +}; + +/* Multifd zlib compression */ + +/** + * zlib_send_setup: setup send side + * + * Setup each channel with zlib compression. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @errp: pointer to an error + */ +static int zlib_send_setup(MultiFDSendParams *p, Error **errp) +{ + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + struct zlib_data *z = g_malloc0(sizeof(struct zlib_data)); + z_stream *zs = &z->zs; + + zs->zalloc = Z_NULL; + zs->zfree = Z_NULL; + zs->opaque = Z_NULL; + if (deflateInit(zs, migrate_multifd_zlib_level()) != Z_OK) { + g_free(z); + error_setg(errp, "multifd %d: deflate init failed", p->id); + return -1; + } + /* We will never have more than page_count pages */ + z->zbuff_len = page_count * qemu_target_page_size(); + z->zbuff_len *= 2; + z->zbuff = g_try_malloc(z->zbuff_len); + if (!z->zbuff) { + deflateEnd(&z->zs); + g_free(z); + error_setg(errp, "multifd %d: out of memory for zbuff", p->id); + return -1; + } + p->data = z; + return 0; +} + +/** + * zlib_send_cleanup: cleanup send side + * + * Close the channel and return memory. + * + * @p: Params for the channel that we are using + */ +static void zlib_send_cleanup(MultiFDSendParams *p, Error **errp) +{ + struct zlib_data *z = p->data; + + deflateEnd(&z->zs); + g_free(z->zbuff); + z->zbuff = NULL; + g_free(p->data); + p->data = NULL; +} + +/** + * zlib_send_prepare: prepare date to be able to send + * + * Create a compressed buffer with all the pages that we are going to + * send. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + */ +static int zlib_send_prepare(MultiFDSendParams *p, uint32_t used, Error **errp) +{ + struct iovec *iov = p->pages->iov; + struct zlib_data *z = p->data; + z_stream *zs = &z->zs; + uint32_t out_size = 0; + int ret; + uint32_t i; + + for (i = 0; i < used; i++) { + uint32_t available = z->zbuff_len - out_size; + int flush = Z_NO_FLUSH; + + if (i == used - 1) { + flush = Z_SYNC_FLUSH; + } + + zs->avail_in = iov[i].iov_len; + zs->next_in = iov[i].iov_base; + + zs->avail_out = available; + zs->next_out = z->zbuff + out_size; + + /* + * Welcome to deflate semantics + * + * We need to loop while: + * - return is Z_OK + * - there are stuff to be compressed + * - there are output space free + */ + do { + ret = deflate(zs, flush); + } while (ret == Z_OK && zs->avail_in && zs->avail_out); + if (ret == Z_OK && zs->avail_in) { + error_setg(errp, "multifd %d: deflate failed to compress all input", + p->id); + return -1; + } + if (ret != Z_OK) { + error_setg(errp, "multifd %d: deflate returned %d instead of Z_OK", + p->id, ret); + return -1; + } + out_size += available - zs->avail_out; + } + p->next_packet_size = out_size; + p->flags |= MULTIFD_FLAG_ZLIB; + + return 0; +} + +/** + * zlib_send_write: do the actual write of the data + * + * Do the actual write of the comprresed buffer. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int zlib_send_write(MultiFDSendParams *p, uint32_t used, Error **errp) +{ + struct zlib_data *z = p->data; + + return qio_channel_write_all(p->c, (void *)z->zbuff, p->next_packet_size, + errp); +} + +/** + * zlib_recv_setup: setup receive side + * + * Create the compressed channel and buffer. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @errp: pointer to an error + */ +static int zlib_recv_setup(MultiFDRecvParams *p, Error **errp) +{ + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + struct zlib_data *z = g_malloc0(sizeof(struct zlib_data)); + z_stream *zs = &z->zs; + + p->data = z; + zs->zalloc = Z_NULL; + zs->zfree = Z_NULL; + zs->opaque = Z_NULL; + zs->avail_in = 0; + zs->next_in = Z_NULL; + if (inflateInit(zs) != Z_OK) { + error_setg(errp, "multifd %d: inflate init failed", p->id); + return -1; + } + /* We will never have more than page_count pages */ + z->zbuff_len = page_count * qemu_target_page_size(); + /* We know compression "could" use more space */ + z->zbuff_len *= 2; + z->zbuff = g_try_malloc(z->zbuff_len); + if (!z->zbuff) { + inflateEnd(zs); + error_setg(errp, "multifd %d: out of memory for zbuff", p->id); + return -1; + } + return 0; +} + +/** + * zlib_recv_cleanup: setup receive side + * + * For no compression this function does nothing. + * + * @p: Params for the channel that we are using + */ +static void zlib_recv_cleanup(MultiFDRecvParams *p) +{ + struct zlib_data *z = p->data; + + inflateEnd(&z->zs); + g_free(z->zbuff); + z->zbuff = NULL; + g_free(p->data); + p->data = NULL; +} + +/** + * zlib_recv_pages: read the data from the channel into actual pages + * + * Read the compressed buffer, and uncompress it into the actual + * pages. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int zlib_recv_pages(MultiFDRecvParams *p, uint32_t used, Error **errp) +{ + struct zlib_data *z = p->data; + z_stream *zs = &z->zs; + uint32_t in_size = p->next_packet_size; + /* we measure the change of total_out */ + uint32_t out_size = zs->total_out; + uint32_t expected_size = used * qemu_target_page_size(); + uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; + int ret; + int i; + + if (flags != MULTIFD_FLAG_ZLIB) { + error_setg(errp, "multifd %d: flags received %x flags expected %x", + p->id, flags, MULTIFD_FLAG_ZLIB); + return -1; + } + ret = qio_channel_read_all(p->c, (void *)z->zbuff, in_size, errp); + + if (ret != 0) { + return ret; + } + + zs->avail_in = in_size; + zs->next_in = z->zbuff; + + for (i = 0; i < used; i++) { + struct iovec *iov = &p->pages->iov[i]; + int flush = Z_NO_FLUSH; + unsigned long start = zs->total_out; + + if (i == used - 1) { + flush = Z_SYNC_FLUSH; + } + + zs->avail_out = iov->iov_len; + zs->next_out = iov->iov_base; + + /* + * Welcome to inflate semantics + * + * We need to loop while: + * - return is Z_OK + * - there are input available + * - we haven't completed a full page + */ + do { + ret = inflate(zs, flush); + } while (ret == Z_OK && zs->avail_in + && (zs->total_out - start) < iov->iov_len); + if (ret == Z_OK && (zs->total_out - start) < iov->iov_len) { + error_setg(errp, "multifd %d: inflate generated too few output", + p->id); + return -1; + } + if (ret != Z_OK) { + error_setg(errp, "multifd %d: inflate returned %d instead of Z_OK", + p->id, ret); + return -1; + } + } + out_size = zs->total_out - out_size; + if (out_size != expected_size) { + error_setg(errp, "multifd %d: packet size received %d size expected %d", + p->id, out_size, expected_size); + return -1; + } + return 0; +} + +static MultiFDMethods multifd_zlib_ops = { + .send_setup = zlib_send_setup, + .send_cleanup = zlib_send_cleanup, + .send_prepare = zlib_send_prepare, + .send_write = zlib_send_write, + .recv_setup = zlib_recv_setup, + .recv_cleanup = zlib_recv_cleanup, + .recv_pages = zlib_recv_pages +}; + +static void multifd_zlib_register(void) +{ + multifd_register_ops(MULTIFD_COMPRESSION_ZLIB, &multifd_zlib_ops); +} + +migration_init(multifd_zlib_register); diff --git a/migration/multifd-zstd.c b/migration/multifd-zstd.c new file mode 100644 index 000000000..693bddf8c --- /dev/null +++ b/migration/multifd-zstd.c @@ -0,0 +1,339 @@ +/* + * Multifd zlib compression implementation + * + * Copyright (c) 2020 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include <zstd.h> +#include "qemu/rcu.h" +#include "exec/target_page.h" +#include "qapi/error.h" +#include "migration.h" +#include "trace.h" +#include "multifd.h" + +struct zstd_data { + /* stream for compression */ + ZSTD_CStream *zcs; + /* stream for decompression */ + ZSTD_DStream *zds; + /* buffers */ + ZSTD_inBuffer in; + ZSTD_outBuffer out; + /* compressed buffer */ + uint8_t *zbuff; + /* size of compressed buffer */ + uint32_t zbuff_len; +}; + +/* Multifd zstd compression */ + +/** + * zstd_send_setup: setup send side + * + * Setup each channel with zstd compression. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @errp: pointer to an error + */ +static int zstd_send_setup(MultiFDSendParams *p, Error **errp) +{ + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + struct zstd_data *z = g_new0(struct zstd_data, 1); + int res; + + p->data = z; + z->zcs = ZSTD_createCStream(); + if (!z->zcs) { + g_free(z); + error_setg(errp, "multifd %d: zstd createCStream failed", p->id); + return -1; + } + + res = ZSTD_initCStream(z->zcs, migrate_multifd_zstd_level()); + if (ZSTD_isError(res)) { + ZSTD_freeCStream(z->zcs); + g_free(z); + error_setg(errp, "multifd %d: initCStream failed with error %s", + p->id, ZSTD_getErrorName(res)); + return -1; + } + /* We will never have more than page_count pages */ + z->zbuff_len = page_count * qemu_target_page_size(); + z->zbuff_len *= 2; + z->zbuff = g_try_malloc(z->zbuff_len); + if (!z->zbuff) { + ZSTD_freeCStream(z->zcs); + g_free(z); + error_setg(errp, "multifd %d: out of memory for zbuff", p->id); + return -1; + } + return 0; +} + +/** + * zstd_send_cleanup: cleanup send side + * + * Close the channel and return memory. + * + * @p: Params for the channel that we are using + */ +static void zstd_send_cleanup(MultiFDSendParams *p, Error **errp) +{ + struct zstd_data *z = p->data; + + ZSTD_freeCStream(z->zcs); + z->zcs = NULL; + g_free(z->zbuff); + z->zbuff = NULL; + g_free(p->data); + p->data = NULL; +} + +/** + * zstd_send_prepare: prepare date to be able to send + * + * Create a compressed buffer with all the pages that we are going to + * send. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + */ +static int zstd_send_prepare(MultiFDSendParams *p, uint32_t used, Error **errp) +{ + struct iovec *iov = p->pages->iov; + struct zstd_data *z = p->data; + int ret; + uint32_t i; + + z->out.dst = z->zbuff; + z->out.size = z->zbuff_len; + z->out.pos = 0; + + for (i = 0; i < used; i++) { + ZSTD_EndDirective flush = ZSTD_e_continue; + + if (i == used - 1) { + flush = ZSTD_e_flush; + } + z->in.src = iov[i].iov_base; + z->in.size = iov[i].iov_len; + z->in.pos = 0; + + /* + * Welcome to compressStream2 semantics + * + * We need to loop while: + * - return is > 0 + * - there is input available + * - there is output space free + */ + do { + ret = ZSTD_compressStream2(z->zcs, &z->out, &z->in, flush); + } while (ret > 0 && (z->in.size - z->in.pos > 0) + && (z->out.size - z->out.pos > 0)); + if (ret > 0 && (z->in.size - z->in.pos > 0)) { + error_setg(errp, "multifd %d: compressStream buffer too small", + p->id); + return -1; + } + if (ZSTD_isError(ret)) { + error_setg(errp, "multifd %d: compressStream error %s", + p->id, ZSTD_getErrorName(ret)); + return -1; + } + } + p->next_packet_size = z->out.pos; + p->flags |= MULTIFD_FLAG_ZSTD; + + return 0; +} + +/** + * zstd_send_write: do the actual write of the data + * + * Do the actual write of the comprresed buffer. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int zstd_send_write(MultiFDSendParams *p, uint32_t used, Error **errp) +{ + struct zstd_data *z = p->data; + + return qio_channel_write_all(p->c, (void *)z->zbuff, p->next_packet_size, + errp); +} + +/** + * zstd_recv_setup: setup receive side + * + * Create the compressed channel and buffer. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @errp: pointer to an error + */ +static int zstd_recv_setup(MultiFDRecvParams *p, Error **errp) +{ + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + struct zstd_data *z = g_new0(struct zstd_data, 1); + int ret; + + p->data = z; + z->zds = ZSTD_createDStream(); + if (!z->zds) { + g_free(z); + error_setg(errp, "multifd %d: zstd createDStream failed", p->id); + return -1; + } + + ret = ZSTD_initDStream(z->zds); + if (ZSTD_isError(ret)) { + ZSTD_freeDStream(z->zds); + g_free(z); + error_setg(errp, "multifd %d: initDStream failed with error %s", + p->id, ZSTD_getErrorName(ret)); + return -1; + } + + /* We will never have more than page_count pages */ + z->zbuff_len = page_count * qemu_target_page_size(); + /* We know compression "could" use more space */ + z->zbuff_len *= 2; + z->zbuff = g_try_malloc(z->zbuff_len); + if (!z->zbuff) { + ZSTD_freeDStream(z->zds); + g_free(z); + error_setg(errp, "multifd %d: out of memory for zbuff", p->id); + return -1; + } + return 0; +} + +/** + * zstd_recv_cleanup: setup receive side + * + * For no compression this function does nothing. + * + * @p: Params for the channel that we are using + */ +static void zstd_recv_cleanup(MultiFDRecvParams *p) +{ + struct zstd_data *z = p->data; + + ZSTD_freeDStream(z->zds); + z->zds = NULL; + g_free(z->zbuff); + z->zbuff = NULL; + g_free(p->data); + p->data = NULL; +} + +/** + * zstd_recv_pages: read the data from the channel into actual pages + * + * Read the compressed buffer, and uncompress it into the actual + * pages. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int zstd_recv_pages(MultiFDRecvParams *p, uint32_t used, Error **errp) +{ + uint32_t in_size = p->next_packet_size; + uint32_t out_size = 0; + uint32_t expected_size = used * qemu_target_page_size(); + uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; + struct zstd_data *z = p->data; + int ret; + int i; + + if (flags != MULTIFD_FLAG_ZSTD) { + error_setg(errp, "multifd %d: flags received %x flags expected %x", + p->id, flags, MULTIFD_FLAG_ZSTD); + return -1; + } + ret = qio_channel_read_all(p->c, (void *)z->zbuff, in_size, errp); + + if (ret != 0) { + return ret; + } + + z->in.src = z->zbuff; + z->in.size = in_size; + z->in.pos = 0; + + for (i = 0; i < used; i++) { + struct iovec *iov = &p->pages->iov[i]; + + z->out.dst = iov->iov_base; + z->out.size = iov->iov_len; + z->out.pos = 0; + + /* + * Welcome to decompressStream semantics + * + * We need to loop while: + * - return is > 0 + * - there is input available + * - we haven't put out a full page + */ + do { + ret = ZSTD_decompressStream(z->zds, &z->out, &z->in); + } while (ret > 0 && (z->in.size - z->in.pos > 0) + && (z->out.pos < iov->iov_len)); + if (ret > 0 && (z->out.pos < iov->iov_len)) { + error_setg(errp, "multifd %d: decompressStream buffer too small", + p->id); + return -1; + } + if (ZSTD_isError(ret)) { + error_setg(errp, "multifd %d: decompressStream returned %s", + p->id, ZSTD_getErrorName(ret)); + return ret; + } + out_size += z->out.pos; + } + if (out_size != expected_size) { + error_setg(errp, "multifd %d: packet size received %d size expected %d", + p->id, out_size, expected_size); + return -1; + } + return 0; +} + +static MultiFDMethods multifd_zstd_ops = { + .send_setup = zstd_send_setup, + .send_cleanup = zstd_send_cleanup, + .send_prepare = zstd_send_prepare, + .send_write = zstd_send_write, + .recv_setup = zstd_recv_setup, + .recv_cleanup = zstd_recv_cleanup, + .recv_pages = zstd_recv_pages +}; + +static void multifd_zstd_register(void) +{ + multifd_register_ops(MULTIFD_COMPRESSION_ZSTD, &multifd_zstd_ops); +} + +migration_init(multifd_zstd_register); diff --git a/migration/multifd.c b/migration/multifd.c new file mode 100644 index 000000000..7c9deb192 --- /dev/null +++ b/migration/multifd.c @@ -0,0 +1,1236 @@ +/* + * Multifd common code + * + * Copyright (c) 2019-2020 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu/rcu.h" +#include "exec/target_page.h" +#include "sysemu/sysemu.h" +#include "exec/ramblock.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "ram.h" +#include "migration.h" +#include "socket.h" +#include "tls.h" +#include "qemu-file.h" +#include "trace.h" +#include "multifd.h" + +#include "qemu/yank.h" +#include "io/channel-socket.h" +#include "yank_functions.h" + +/* Multiple fd's */ + +#define MULTIFD_MAGIC 0x11223344U +#define MULTIFD_VERSION 1 + +typedef struct { + uint32_t magic; + uint32_t version; + unsigned char uuid[16]; /* QemuUUID */ + uint8_t id; + uint8_t unused1[7]; /* Reserved for future use */ + uint64_t unused2[4]; /* Reserved for future use */ +} __attribute__((packed)) MultiFDInit_t; + +/* Multifd without compression */ + +/** + * nocomp_send_setup: setup send side + * + * For no compression this function does nothing. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @errp: pointer to an error + */ +static int nocomp_send_setup(MultiFDSendParams *p, Error **errp) +{ + return 0; +} + +/** + * nocomp_send_cleanup: cleanup send side + * + * For no compression this function does nothing. + * + * @p: Params for the channel that we are using + */ +static void nocomp_send_cleanup(MultiFDSendParams *p, Error **errp) +{ + return; +} + +/** + * nocomp_send_prepare: prepare date to be able to send + * + * For no compression we just have to calculate the size of the + * packet. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int nocomp_send_prepare(MultiFDSendParams *p, uint32_t used, + Error **errp) +{ + p->next_packet_size = used * qemu_target_page_size(); + p->flags |= MULTIFD_FLAG_NOCOMP; + return 0; +} + +/** + * nocomp_send_write: do the actual write of the data + * + * For no compression we just have to write the data. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int nocomp_send_write(MultiFDSendParams *p, uint32_t used, Error **errp) +{ + return qio_channel_writev_all(p->c, p->pages->iov, used, errp); +} + +/** + * nocomp_recv_setup: setup receive side + * + * For no compression this function does nothing. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @errp: pointer to an error + */ +static int nocomp_recv_setup(MultiFDRecvParams *p, Error **errp) +{ + return 0; +} + +/** + * nocomp_recv_cleanup: setup receive side + * + * For no compression this function does nothing. + * + * @p: Params for the channel that we are using + */ +static void nocomp_recv_cleanup(MultiFDRecvParams *p) +{ +} + +/** + * nocomp_recv_pages: read the data from the channel into actual pages + * + * For no compression we just need to read things into the correct place. + * + * Returns 0 for success or -1 for error + * + * @p: Params for the channel that we are using + * @used: number of pages used + * @errp: pointer to an error + */ +static int nocomp_recv_pages(MultiFDRecvParams *p, uint32_t used, Error **errp) +{ + uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; + + if (flags != MULTIFD_FLAG_NOCOMP) { + error_setg(errp, "multifd %d: flags received %x flags expected %x", + p->id, flags, MULTIFD_FLAG_NOCOMP); + return -1; + } + return qio_channel_readv_all(p->c, p->pages->iov, used, errp); +} + +static MultiFDMethods multifd_nocomp_ops = { + .send_setup = nocomp_send_setup, + .send_cleanup = nocomp_send_cleanup, + .send_prepare = nocomp_send_prepare, + .send_write = nocomp_send_write, + .recv_setup = nocomp_recv_setup, + .recv_cleanup = nocomp_recv_cleanup, + .recv_pages = nocomp_recv_pages +}; + +static MultiFDMethods *multifd_ops[MULTIFD_COMPRESSION__MAX] = { + [MULTIFD_COMPRESSION_NONE] = &multifd_nocomp_ops, +}; + +void multifd_register_ops(int method, MultiFDMethods *ops) +{ + assert(0 < method && method < MULTIFD_COMPRESSION__MAX); + multifd_ops[method] = ops; +} + +static int multifd_send_initial_packet(MultiFDSendParams *p, Error **errp) +{ + MultiFDInit_t msg = {}; + int ret; + + msg.magic = cpu_to_be32(MULTIFD_MAGIC); + msg.version = cpu_to_be32(MULTIFD_VERSION); + msg.id = p->id; + memcpy(msg.uuid, &qemu_uuid.data, sizeof(msg.uuid)); + + ret = qio_channel_write_all(p->c, (char *)&msg, sizeof(msg), errp); + if (ret != 0) { + return -1; + } + return 0; +} + +static int multifd_recv_initial_packet(QIOChannel *c, Error **errp) +{ + MultiFDInit_t msg; + int ret; + + ret = qio_channel_read_all(c, (char *)&msg, sizeof(msg), errp); + if (ret != 0) { + return -1; + } + + msg.magic = be32_to_cpu(msg.magic); + msg.version = be32_to_cpu(msg.version); + + if (msg.magic != MULTIFD_MAGIC) { + error_setg(errp, "multifd: received packet magic %x " + "expected %x", msg.magic, MULTIFD_MAGIC); + return -1; + } + + if (msg.version != MULTIFD_VERSION) { + error_setg(errp, "multifd: received packet version %d " + "expected %d", msg.version, MULTIFD_VERSION); + return -1; + } + + if (memcmp(msg.uuid, &qemu_uuid, sizeof(qemu_uuid))) { + char *uuid = qemu_uuid_unparse_strdup(&qemu_uuid); + char *msg_uuid = qemu_uuid_unparse_strdup((const QemuUUID *)msg.uuid); + + error_setg(errp, "multifd: received uuid '%s' and expected " + "uuid '%s' for channel %hhd", msg_uuid, uuid, msg.id); + g_free(uuid); + g_free(msg_uuid); + return -1; + } + + if (msg.id > migrate_multifd_channels()) { + error_setg(errp, "multifd: received channel version %d " + "expected %d", msg.version, MULTIFD_VERSION); + return -1; + } + + return msg.id; +} + +static MultiFDPages_t *multifd_pages_init(size_t size) +{ + MultiFDPages_t *pages = g_new0(MultiFDPages_t, 1); + + pages->allocated = size; + pages->iov = g_new0(struct iovec, size); + pages->offset = g_new0(ram_addr_t, size); + + return pages; +} + +static void multifd_pages_clear(MultiFDPages_t *pages) +{ + pages->used = 0; + pages->allocated = 0; + pages->packet_num = 0; + pages->block = NULL; + g_free(pages->iov); + pages->iov = NULL; + g_free(pages->offset); + pages->offset = NULL; + g_free(pages); +} + +static void multifd_send_fill_packet(MultiFDSendParams *p) +{ + MultiFDPacket_t *packet = p->packet; + int i; + + packet->flags = cpu_to_be32(p->flags); + packet->pages_alloc = cpu_to_be32(p->pages->allocated); + packet->pages_used = cpu_to_be32(p->pages->used); + packet->next_packet_size = cpu_to_be32(p->next_packet_size); + packet->packet_num = cpu_to_be64(p->packet_num); + + if (p->pages->block) { + strncpy(packet->ramblock, p->pages->block->idstr, 256); + } + + for (i = 0; i < p->pages->used; i++) { + /* there are architectures where ram_addr_t is 32 bit */ + uint64_t temp = p->pages->offset[i]; + + packet->offset[i] = cpu_to_be64(temp); + } +} + +static int multifd_recv_unfill_packet(MultiFDRecvParams *p, Error **errp) +{ + MultiFDPacket_t *packet = p->packet; + uint32_t pages_max = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + RAMBlock *block; + int i; + + packet->magic = be32_to_cpu(packet->magic); + if (packet->magic != MULTIFD_MAGIC) { + error_setg(errp, "multifd: received packet " + "magic %x and expected magic %x", + packet->magic, MULTIFD_MAGIC); + return -1; + } + + packet->version = be32_to_cpu(packet->version); + if (packet->version != MULTIFD_VERSION) { + error_setg(errp, "multifd: received packet " + "version %d and expected version %d", + packet->version, MULTIFD_VERSION); + return -1; + } + + p->flags = be32_to_cpu(packet->flags); + + packet->pages_alloc = be32_to_cpu(packet->pages_alloc); + /* + * If we received a packet that is 100 times bigger than expected + * just stop migration. It is a magic number. + */ + if (packet->pages_alloc > pages_max * 100) { + error_setg(errp, "multifd: received packet " + "with size %d and expected a maximum size of %d", + packet->pages_alloc, pages_max * 100) ; + return -1; + } + /* + * We received a packet that is bigger than expected but inside + * reasonable limits (see previous comment). Just reallocate. + */ + if (packet->pages_alloc > p->pages->allocated) { + multifd_pages_clear(p->pages); + p->pages = multifd_pages_init(packet->pages_alloc); + } + + p->pages->used = be32_to_cpu(packet->pages_used); + if (p->pages->used > packet->pages_alloc) { + error_setg(errp, "multifd: received packet " + "with %d pages and expected maximum pages are %d", + p->pages->used, packet->pages_alloc) ; + return -1; + } + + p->next_packet_size = be32_to_cpu(packet->next_packet_size); + p->packet_num = be64_to_cpu(packet->packet_num); + + if (p->pages->used == 0) { + return 0; + } + + /* make sure that ramblock is 0 terminated */ + packet->ramblock[255] = 0; + block = qemu_ram_block_by_name(packet->ramblock); + if (!block) { + error_setg(errp, "multifd: unknown ram block %s", + packet->ramblock); + return -1; + } + + for (i = 0; i < p->pages->used; i++) { + uint64_t offset = be64_to_cpu(packet->offset[i]); + + if (offset > (block->used_length - qemu_target_page_size())) { + error_setg(errp, "multifd: offset too long %" PRIu64 + " (max " RAM_ADDR_FMT ")", + offset, block->used_length); + return -1; + } + p->pages->iov[i].iov_base = block->host + offset; + p->pages->iov[i].iov_len = qemu_target_page_size(); + } + + return 0; +} + +struct { + MultiFDSendParams *params; + /* array of pages to sent */ + MultiFDPages_t *pages; + /* global number of generated multifd packets */ + uint64_t packet_num; + /* send channels ready */ + QemuSemaphore channels_ready; + /* + * Have we already run terminate threads. There is a race when it + * happens that we got one error while we are exiting. + * We will use atomic operations. Only valid values are 0 and 1. + */ + int exiting; + /* multifd ops */ + MultiFDMethods *ops; +} *multifd_send_state; + +/* + * How we use multifd_send_state->pages and channel->pages? + * + * We create a pages for each channel, and a main one. Each time that + * we need to send a batch of pages we interchange the ones between + * multifd_send_state and the channel that is sending it. There are + * two reasons for that: + * - to not have to do so many mallocs during migration + * - to make easier to know what to free at the end of migration + * + * This way we always know who is the owner of each "pages" struct, + * and we don't need any locking. It belongs to the migration thread + * or to the channel thread. Switching is safe because the migration + * thread is using the channel mutex when changing it, and the channel + * have to had finish with its own, otherwise pending_job can't be + * false. + */ + +static int multifd_send_pages(QEMUFile *f) +{ + int i; + static int next_channel; + MultiFDSendParams *p = NULL; /* make happy gcc */ + MultiFDPages_t *pages = multifd_send_state->pages; + uint64_t transferred; + + if (qatomic_read(&multifd_send_state->exiting)) { + return -1; + } + + qemu_sem_wait(&multifd_send_state->channels_ready); + /* + * next_channel can remain from a previous migration that was + * using more channels, so ensure it doesn't overflow if the + * limit is lower now. + */ + next_channel %= migrate_multifd_channels(); + for (i = next_channel;; i = (i + 1) % migrate_multifd_channels()) { + p = &multifd_send_state->params[i]; + + qemu_mutex_lock(&p->mutex); + if (p->quit) { + error_report("%s: channel %d has already quit!", __func__, i); + qemu_mutex_unlock(&p->mutex); + return -1; + } + if (!p->pending_job) { + p->pending_job++; + next_channel = (i + 1) % migrate_multifd_channels(); + break; + } + qemu_mutex_unlock(&p->mutex); + } + assert(!p->pages->used); + assert(!p->pages->block); + + p->packet_num = multifd_send_state->packet_num++; + multifd_send_state->pages = p->pages; + p->pages = pages; + transferred = ((uint64_t) pages->used) * qemu_target_page_size() + + p->packet_len; + qemu_file_update_transfer(f, transferred); + ram_counters.multifd_bytes += transferred; + ram_counters.transferred += transferred; + qemu_mutex_unlock(&p->mutex); + qemu_sem_post(&p->sem); + + return 1; +} + +int multifd_queue_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset) +{ + MultiFDPages_t *pages = multifd_send_state->pages; + + if (!pages->block) { + pages->block = block; + } + + if (pages->block == block) { + pages->offset[pages->used] = offset; + pages->iov[pages->used].iov_base = block->host + offset; + pages->iov[pages->used].iov_len = qemu_target_page_size(); + pages->used++; + + if (pages->used < pages->allocated) { + return 1; + } + } + + if (multifd_send_pages(f) < 0) { + return -1; + } + + if (pages->block != block) { + return multifd_queue_page(f, block, offset); + } + + return 1; +} + +static void multifd_send_terminate_threads(Error *err) +{ + int i; + + trace_multifd_send_terminate_threads(err != NULL); + + if (err) { + MigrationState *s = migrate_get_current(); + migrate_set_error(s, err); + if (s->state == MIGRATION_STATUS_SETUP || + s->state == MIGRATION_STATUS_PRE_SWITCHOVER || + s->state == MIGRATION_STATUS_DEVICE || + s->state == MIGRATION_STATUS_ACTIVE) { + migrate_set_state(&s->state, s->state, + MIGRATION_STATUS_FAILED); + } + } + + /* + * We don't want to exit each threads twice. Depending on where + * we get the error, or if there are two independent errors in two + * threads at the same time, we can end calling this function + * twice. + */ + if (qatomic_xchg(&multifd_send_state->exiting, 1)) { + return; + } + + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + + qemu_mutex_lock(&p->mutex); + p->quit = true; + qemu_sem_post(&p->sem); + qemu_mutex_unlock(&p->mutex); + } +} + +void multifd_save_cleanup(void) +{ + int i; + + if (!migrate_use_multifd() || !migrate_multifd_is_allowed()) { + return; + } + multifd_send_terminate_threads(NULL); + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + + if (p->running) { + qemu_thread_join(&p->thread); + } + } + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + Error *local_err = NULL; + + if (p->registered_yank) { + migration_ioc_unregister_yank(p->c); + } + socket_send_channel_destroy(p->c); + p->c = NULL; + qemu_mutex_destroy(&p->mutex); + qemu_sem_destroy(&p->sem); + qemu_sem_destroy(&p->sem_sync); + g_free(p->name); + p->name = NULL; + g_free(p->tls_hostname); + p->tls_hostname = NULL; + multifd_pages_clear(p->pages); + p->pages = NULL; + p->packet_len = 0; + g_free(p->packet); + p->packet = NULL; + multifd_send_state->ops->send_cleanup(p, &local_err); + if (local_err) { + migrate_set_error(migrate_get_current(), local_err); + error_free(local_err); + } + } + qemu_sem_destroy(&multifd_send_state->channels_ready); + g_free(multifd_send_state->params); + multifd_send_state->params = NULL; + multifd_pages_clear(multifd_send_state->pages); + multifd_send_state->pages = NULL; + g_free(multifd_send_state); + multifd_send_state = NULL; +} + +void multifd_send_sync_main(QEMUFile *f) +{ + int i; + + if (!migrate_use_multifd()) { + return; + } + if (multifd_send_state->pages->used) { + if (multifd_send_pages(f) < 0) { + error_report("%s: multifd_send_pages fail", __func__); + return; + } + } + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + + trace_multifd_send_sync_main_signal(p->id); + + qemu_mutex_lock(&p->mutex); + + if (p->quit) { + error_report("%s: channel %d has already quit", __func__, i); + qemu_mutex_unlock(&p->mutex); + return; + } + + p->packet_num = multifd_send_state->packet_num++; + p->flags |= MULTIFD_FLAG_SYNC; + p->pending_job++; + qemu_file_update_transfer(f, p->packet_len); + ram_counters.multifd_bytes += p->packet_len; + ram_counters.transferred += p->packet_len; + qemu_mutex_unlock(&p->mutex); + qemu_sem_post(&p->sem); + } + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + + trace_multifd_send_sync_main_wait(p->id); + qemu_sem_wait(&p->sem_sync); + } + trace_multifd_send_sync_main(multifd_send_state->packet_num); +} + +static void *multifd_send_thread(void *opaque) +{ + MultiFDSendParams *p = opaque; + Error *local_err = NULL; + int ret = 0; + uint32_t flags = 0; + + trace_multifd_send_thread_start(p->id); + rcu_register_thread(); + + if (multifd_send_initial_packet(p, &local_err) < 0) { + ret = -1; + goto out; + } + /* initial packet */ + p->num_packets = 1; + + while (true) { + qemu_sem_wait(&p->sem); + + if (qatomic_read(&multifd_send_state->exiting)) { + break; + } + qemu_mutex_lock(&p->mutex); + + if (p->pending_job) { + uint32_t used = p->pages->used; + uint64_t packet_num = p->packet_num; + flags = p->flags; + + if (used) { + ret = multifd_send_state->ops->send_prepare(p, used, + &local_err); + if (ret != 0) { + qemu_mutex_unlock(&p->mutex); + break; + } + } + multifd_send_fill_packet(p); + p->flags = 0; + p->num_packets++; + p->num_pages += used; + p->pages->used = 0; + p->pages->block = NULL; + qemu_mutex_unlock(&p->mutex); + + trace_multifd_send(p->id, packet_num, used, flags, + p->next_packet_size); + + ret = qio_channel_write_all(p->c, (void *)p->packet, + p->packet_len, &local_err); + if (ret != 0) { + break; + } + + if (used) { + ret = multifd_send_state->ops->send_write(p, used, &local_err); + if (ret != 0) { + break; + } + } + + qemu_mutex_lock(&p->mutex); + p->pending_job--; + qemu_mutex_unlock(&p->mutex); + + if (flags & MULTIFD_FLAG_SYNC) { + qemu_sem_post(&p->sem_sync); + } + qemu_sem_post(&multifd_send_state->channels_ready); + } else if (p->quit) { + qemu_mutex_unlock(&p->mutex); + break; + } else { + qemu_mutex_unlock(&p->mutex); + /* sometimes there are spurious wakeups */ + } + } + +out: + if (local_err) { + trace_multifd_send_error(p->id); + multifd_send_terminate_threads(local_err); + error_free(local_err); + } + + /* + * Error happen, I will exit, but I can't just leave, tell + * who pay attention to me. + */ + if (ret != 0) { + qemu_sem_post(&p->sem_sync); + qemu_sem_post(&multifd_send_state->channels_ready); + } + + qemu_mutex_lock(&p->mutex); + p->running = false; + qemu_mutex_unlock(&p->mutex); + + rcu_unregister_thread(); + trace_multifd_send_thread_end(p->id, p->num_packets, p->num_pages); + + return NULL; +} + +static bool multifd_channel_connect(MultiFDSendParams *p, + QIOChannel *ioc, + Error *error); + +static void multifd_tls_outgoing_handshake(QIOTask *task, + gpointer opaque) +{ + MultiFDSendParams *p = opaque; + QIOChannel *ioc = QIO_CHANNEL(qio_task_get_source(task)); + Error *err = NULL; + + if (qio_task_propagate_error(task, &err)) { + trace_multifd_tls_outgoing_handshake_error(ioc, error_get_pretty(err)); + } else { + trace_multifd_tls_outgoing_handshake_complete(ioc); + } + + if (!multifd_channel_connect(p, ioc, err)) { + /* + * Error happen, mark multifd_send_thread status as 'quit' although it + * is not created, and then tell who pay attention to me. + */ + p->quit = true; + qemu_sem_post(&multifd_send_state->channels_ready); + qemu_sem_post(&p->sem_sync); + } +} + +static void *multifd_tls_handshake_thread(void *opaque) +{ + MultiFDSendParams *p = opaque; + QIOChannelTLS *tioc = QIO_CHANNEL_TLS(p->c); + + qio_channel_tls_handshake(tioc, + multifd_tls_outgoing_handshake, + p, + NULL, + NULL); + return NULL; +} + +static void multifd_tls_channel_connect(MultiFDSendParams *p, + QIOChannel *ioc, + Error **errp) +{ + MigrationState *s = migrate_get_current(); + const char *hostname = p->tls_hostname; + QIOChannelTLS *tioc; + + tioc = migration_tls_client_create(s, ioc, hostname, errp); + if (!tioc) { + return; + } + + object_unref(OBJECT(ioc)); + trace_multifd_tls_outgoing_handshake_start(ioc, tioc, hostname); + qio_channel_set_name(QIO_CHANNEL(tioc), "multifd-tls-outgoing"); + p->c = QIO_CHANNEL(tioc); + qemu_thread_create(&p->thread, "multifd-tls-handshake-worker", + multifd_tls_handshake_thread, p, + QEMU_THREAD_JOINABLE); +} + +static bool multifd_channel_connect(MultiFDSendParams *p, + QIOChannel *ioc, + Error *error) +{ + MigrationState *s = migrate_get_current(); + + trace_multifd_set_outgoing_channel( + ioc, object_get_typename(OBJECT(ioc)), p->tls_hostname, error); + + if (!error) { + if (s->parameters.tls_creds && + *s->parameters.tls_creds && + !object_dynamic_cast(OBJECT(ioc), + TYPE_QIO_CHANNEL_TLS)) { + multifd_tls_channel_connect(p, ioc, &error); + if (!error) { + /* + * tls_channel_connect will call back to this + * function after the TLS handshake, + * so we mustn't call multifd_send_thread until then + */ + return true; + } else { + return false; + } + } else { + migration_ioc_register_yank(ioc); + p->registered_yank = true; + p->c = ioc; + qemu_thread_create(&p->thread, p->name, multifd_send_thread, p, + QEMU_THREAD_JOINABLE); + } + return true; + } + + return false; +} + +static void multifd_new_send_channel_cleanup(MultiFDSendParams *p, + QIOChannel *ioc, Error *err) +{ + migrate_set_error(migrate_get_current(), err); + /* Error happen, we need to tell who pay attention to me */ + qemu_sem_post(&multifd_send_state->channels_ready); + qemu_sem_post(&p->sem_sync); + /* + * Although multifd_send_thread is not created, but main migration + * thread neet to judge whether it is running, so we need to mark + * its status. + */ + p->quit = true; + object_unref(OBJECT(ioc)); + error_free(err); +} + +static void multifd_new_send_channel_async(QIOTask *task, gpointer opaque) +{ + MultiFDSendParams *p = opaque; + QIOChannel *sioc = QIO_CHANNEL(qio_task_get_source(task)); + Error *local_err = NULL; + + trace_multifd_new_send_channel_async(p->id); + if (qio_task_propagate_error(task, &local_err)) { + goto cleanup; + } else { + p->c = QIO_CHANNEL(sioc); + qio_channel_set_delay(p->c, false); + p->running = true; + if (!multifd_channel_connect(p, sioc, local_err)) { + goto cleanup; + } + return; + } + +cleanup: + multifd_new_send_channel_cleanup(p, sioc, local_err); +} + +static bool migrate_allow_multifd = true; +void migrate_protocol_allow_multifd(bool allow) +{ + migrate_allow_multifd = allow; +} + +bool migrate_multifd_is_allowed(void) +{ + return migrate_allow_multifd; +} + +int multifd_save_setup(Error **errp) +{ + int thread_count; + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + uint8_t i; + MigrationState *s; + + if (!migrate_use_multifd()) { + return 0; + } + if (!migrate_multifd_is_allowed()) { + error_setg(errp, "multifd is not supported by current protocol"); + return -1; + } + + s = migrate_get_current(); + thread_count = migrate_multifd_channels(); + multifd_send_state = g_malloc0(sizeof(*multifd_send_state)); + multifd_send_state->params = g_new0(MultiFDSendParams, thread_count); + multifd_send_state->pages = multifd_pages_init(page_count); + qemu_sem_init(&multifd_send_state->channels_ready, 0); + qatomic_set(&multifd_send_state->exiting, 0); + multifd_send_state->ops = multifd_ops[migrate_multifd_compression()]; + + for (i = 0; i < thread_count; i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + + qemu_mutex_init(&p->mutex); + qemu_sem_init(&p->sem, 0); + qemu_sem_init(&p->sem_sync, 0); + p->quit = false; + p->pending_job = 0; + p->id = i; + p->pages = multifd_pages_init(page_count); + p->packet_len = sizeof(MultiFDPacket_t) + + sizeof(uint64_t) * page_count; + p->packet = g_malloc0(p->packet_len); + p->packet->magic = cpu_to_be32(MULTIFD_MAGIC); + p->packet->version = cpu_to_be32(MULTIFD_VERSION); + p->name = g_strdup_printf("multifdsend_%d", i); + p->tls_hostname = g_strdup(s->hostname); + socket_send_channel_create(multifd_new_send_channel_async, p); + } + + for (i = 0; i < thread_count; i++) { + MultiFDSendParams *p = &multifd_send_state->params[i]; + Error *local_err = NULL; + int ret; + + ret = multifd_send_state->ops->send_setup(p, &local_err); + if (ret) { + error_propagate(errp, local_err); + return ret; + } + } + return 0; +} + +struct { + MultiFDRecvParams *params; + /* number of created threads */ + int count; + /* syncs main thread and channels */ + QemuSemaphore sem_sync; + /* global number of generated multifd packets */ + uint64_t packet_num; + /* multifd ops */ + MultiFDMethods *ops; +} *multifd_recv_state; + +static void multifd_recv_terminate_threads(Error *err) +{ + int i; + + trace_multifd_recv_terminate_threads(err != NULL); + + if (err) { + MigrationState *s = migrate_get_current(); + migrate_set_error(s, err); + if (s->state == MIGRATION_STATUS_SETUP || + s->state == MIGRATION_STATUS_ACTIVE) { + migrate_set_state(&s->state, s->state, + MIGRATION_STATUS_FAILED); + } + } + + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + + qemu_mutex_lock(&p->mutex); + p->quit = true; + /* + * We could arrive here for two reasons: + * - normal quit, i.e. everything went fine, just finished + * - error quit: We close the channels so the channel threads + * finish the qio_channel_read_all_eof() + */ + if (p->c) { + qio_channel_shutdown(p->c, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); + } + qemu_mutex_unlock(&p->mutex); + } +} + +int multifd_load_cleanup(Error **errp) +{ + int i; + + if (!migrate_use_multifd() || !migrate_multifd_is_allowed()) { + return 0; + } + multifd_recv_terminate_threads(NULL); + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + + if (p->running) { + p->quit = true; + /* + * multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code, + * however try to wakeup it without harm in cleanup phase. + */ + qemu_sem_post(&p->sem_sync); + qemu_thread_join(&p->thread); + } + } + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + + migration_ioc_unregister_yank(p->c); + object_unref(OBJECT(p->c)); + p->c = NULL; + qemu_mutex_destroy(&p->mutex); + qemu_sem_destroy(&p->sem_sync); + g_free(p->name); + p->name = NULL; + multifd_pages_clear(p->pages); + p->pages = NULL; + p->packet_len = 0; + g_free(p->packet); + p->packet = NULL; + multifd_recv_state->ops->recv_cleanup(p); + } + qemu_sem_destroy(&multifd_recv_state->sem_sync); + g_free(multifd_recv_state->params); + multifd_recv_state->params = NULL; + g_free(multifd_recv_state); + multifd_recv_state = NULL; + + return 0; +} + +void multifd_recv_sync_main(void) +{ + int i; + + if (!migrate_use_multifd()) { + return; + } + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + + trace_multifd_recv_sync_main_wait(p->id); + qemu_sem_wait(&multifd_recv_state->sem_sync); + } + for (i = 0; i < migrate_multifd_channels(); i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + + WITH_QEMU_LOCK_GUARD(&p->mutex) { + if (multifd_recv_state->packet_num < p->packet_num) { + multifd_recv_state->packet_num = p->packet_num; + } + } + trace_multifd_recv_sync_main_signal(p->id); + qemu_sem_post(&p->sem_sync); + } + trace_multifd_recv_sync_main(multifd_recv_state->packet_num); +} + +static void *multifd_recv_thread(void *opaque) +{ + MultiFDRecvParams *p = opaque; + Error *local_err = NULL; + int ret; + + trace_multifd_recv_thread_start(p->id); + rcu_register_thread(); + + while (true) { + uint32_t used; + uint32_t flags; + + if (p->quit) { + break; + } + + ret = qio_channel_read_all_eof(p->c, (void *)p->packet, + p->packet_len, &local_err); + if (ret == 0) { /* EOF */ + break; + } + if (ret == -1) { /* Error */ + break; + } + + qemu_mutex_lock(&p->mutex); + ret = multifd_recv_unfill_packet(p, &local_err); + if (ret) { + qemu_mutex_unlock(&p->mutex); + break; + } + + used = p->pages->used; + flags = p->flags; + /* recv methods don't know how to handle the SYNC flag */ + p->flags &= ~MULTIFD_FLAG_SYNC; + trace_multifd_recv(p->id, p->packet_num, used, flags, + p->next_packet_size); + p->num_packets++; + p->num_pages += used; + qemu_mutex_unlock(&p->mutex); + + if (used) { + ret = multifd_recv_state->ops->recv_pages(p, used, &local_err); + if (ret != 0) { + break; + } + } + + if (flags & MULTIFD_FLAG_SYNC) { + qemu_sem_post(&multifd_recv_state->sem_sync); + qemu_sem_wait(&p->sem_sync); + } + } + + if (local_err) { + multifd_recv_terminate_threads(local_err); + error_free(local_err); + } + qemu_mutex_lock(&p->mutex); + p->running = false; + qemu_mutex_unlock(&p->mutex); + + rcu_unregister_thread(); + trace_multifd_recv_thread_end(p->id, p->num_packets, p->num_pages); + + return NULL; +} + +int multifd_load_setup(Error **errp) +{ + int thread_count; + uint32_t page_count = MULTIFD_PACKET_SIZE / qemu_target_page_size(); + uint8_t i; + + if (!migrate_use_multifd()) { + return 0; + } + if (!migrate_multifd_is_allowed()) { + error_setg(errp, "multifd is not supported by current protocol"); + return -1; + } + thread_count = migrate_multifd_channels(); + multifd_recv_state = g_malloc0(sizeof(*multifd_recv_state)); + multifd_recv_state->params = g_new0(MultiFDRecvParams, thread_count); + qatomic_set(&multifd_recv_state->count, 0); + qemu_sem_init(&multifd_recv_state->sem_sync, 0); + multifd_recv_state->ops = multifd_ops[migrate_multifd_compression()]; + + for (i = 0; i < thread_count; i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + + qemu_mutex_init(&p->mutex); + qemu_sem_init(&p->sem_sync, 0); + p->quit = false; + p->id = i; + p->pages = multifd_pages_init(page_count); + p->packet_len = sizeof(MultiFDPacket_t) + + sizeof(uint64_t) * page_count; + p->packet = g_malloc0(p->packet_len); + p->name = g_strdup_printf("multifdrecv_%d", i); + } + + for (i = 0; i < thread_count; i++) { + MultiFDRecvParams *p = &multifd_recv_state->params[i]; + Error *local_err = NULL; + int ret; + + ret = multifd_recv_state->ops->recv_setup(p, &local_err); + if (ret) { + error_propagate(errp, local_err); + return ret; + } + } + return 0; +} + +bool multifd_recv_all_channels_created(void) +{ + int thread_count = migrate_multifd_channels(); + + if (!migrate_use_multifd()) { + return true; + } + + if (!multifd_recv_state) { + /* Called before any connections created */ + return false; + } + + return thread_count == qatomic_read(&multifd_recv_state->count); +} + +/* + * Try to receive all multifd channels to get ready for the migration. + * - Return true and do not set @errp when correctly receiving all channels; + * - Return false and do not set @errp when correctly receiving the current one; + * - Return false and set @errp when failing to receive the current channel. + */ +bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp) +{ + MultiFDRecvParams *p; + Error *local_err = NULL; + int id; + + id = multifd_recv_initial_packet(ioc, &local_err); + if (id < 0) { + multifd_recv_terminate_threads(local_err); + error_propagate_prepend(errp, local_err, + "failed to receive packet" + " via multifd channel %d: ", + qatomic_read(&multifd_recv_state->count)); + return false; + } + trace_multifd_recv_new_channel(id); + + p = &multifd_recv_state->params[id]; + if (p->c != NULL) { + error_setg(&local_err, "multifd: received id '%d' already setup'", + id); + multifd_recv_terminate_threads(local_err); + error_propagate(errp, local_err); + return false; + } + p->c = ioc; + object_ref(OBJECT(ioc)); + /* initial packet */ + p->num_packets = 1; + + p->running = true; + qemu_thread_create(&p->thread, p->name, multifd_recv_thread, p, + QEMU_THREAD_JOINABLE); + qatomic_inc(&multifd_recv_state->count); + return qatomic_read(&multifd_recv_state->count) == + migrate_multifd_channels(); +} diff --git a/migration/multifd.h b/migration/multifd.h new file mode 100644 index 000000000..15c50ca0b --- /dev/null +++ b/migration/multifd.h @@ -0,0 +1,176 @@ +/* + * Multifd common functions + * + * Copyright (c) 2019-2020 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_MIGRATION_MULTIFD_H +#define QEMU_MIGRATION_MULTIFD_H + +bool migrate_multifd_is_allowed(void); +void migrate_protocol_allow_multifd(bool allow); +int multifd_save_setup(Error **errp); +void multifd_save_cleanup(void); +int multifd_load_setup(Error **errp); +int multifd_load_cleanup(Error **errp); +bool multifd_recv_all_channels_created(void); +bool multifd_recv_new_channel(QIOChannel *ioc, Error **errp); +void multifd_recv_sync_main(void); +void multifd_send_sync_main(QEMUFile *f); +int multifd_queue_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset); + +/* Multifd Compression flags */ +#define MULTIFD_FLAG_SYNC (1 << 0) + +/* We reserve 3 bits for compression methods */ +#define MULTIFD_FLAG_COMPRESSION_MASK (7 << 1) +/* we need to be compatible. Before compression value was 0 */ +#define MULTIFD_FLAG_NOCOMP (0 << 1) +#define MULTIFD_FLAG_ZLIB (1 << 1) +#define MULTIFD_FLAG_ZSTD (2 << 1) + +/* This value needs to be a multiple of qemu_target_page_size() */ +#define MULTIFD_PACKET_SIZE (512 * 1024) + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t flags; + /* maximum number of allocated pages */ + uint32_t pages_alloc; + uint32_t pages_used; + /* size of the next packet that contains pages */ + uint32_t next_packet_size; + uint64_t packet_num; + uint64_t unused[4]; /* Reserved for future use */ + char ramblock[256]; + uint64_t offset[]; +} __attribute__((packed)) MultiFDPacket_t; + +typedef struct { + /* number of used pages */ + uint32_t used; + /* number of allocated pages */ + uint32_t allocated; + /* global number of generated multifd packets */ + uint64_t packet_num; + /* offset of each page */ + ram_addr_t *offset; + /* pointer to each page */ + struct iovec *iov; + RAMBlock *block; +} MultiFDPages_t; + +typedef struct { + /* this fields are not changed once the thread is created */ + /* channel number */ + uint8_t id; + /* channel thread name */ + char *name; + /* tls hostname */ + char *tls_hostname; + /* channel thread id */ + QemuThread thread; + /* communication channel */ + QIOChannel *c; + /* sem where to wait for more work */ + QemuSemaphore sem; + /* this mutex protects the following parameters */ + QemuMutex mutex; + /* is this channel thread running */ + bool running; + /* should this thread finish */ + bool quit; + /* is the yank function registered */ + bool registered_yank; + /* thread has work to do */ + int pending_job; + /* array of pages to sent */ + MultiFDPages_t *pages; + /* packet allocated len */ + uint32_t packet_len; + /* pointer to the packet */ + MultiFDPacket_t *packet; + /* multifd flags for each packet */ + uint32_t flags; + /* size of the next packet that contains pages */ + uint32_t next_packet_size; + /* global number of generated multifd packets */ + uint64_t packet_num; + /* thread local variables */ + /* packets sent through this channel */ + uint64_t num_packets; + /* pages sent through this channel */ + uint64_t num_pages; + /* syncs main thread and channels */ + QemuSemaphore sem_sync; + /* used for compression methods */ + void *data; +} MultiFDSendParams; + +typedef struct { + /* this fields are not changed once the thread is created */ + /* channel number */ + uint8_t id; + /* channel thread name */ + char *name; + /* channel thread id */ + QemuThread thread; + /* communication channel */ + QIOChannel *c; + /* this mutex protects the following parameters */ + QemuMutex mutex; + /* is this channel thread running */ + bool running; + /* should this thread finish */ + bool quit; + /* array of pages to receive */ + MultiFDPages_t *pages; + /* packet allocated len */ + uint32_t packet_len; + /* pointer to the packet */ + MultiFDPacket_t *packet; + /* multifd flags for each packet */ + uint32_t flags; + /* global number of generated multifd packets */ + uint64_t packet_num; + /* thread local variables */ + /* size of the next packet that contains pages */ + uint32_t next_packet_size; + /* packets sent through this channel */ + uint64_t num_packets; + /* pages sent through this channel */ + uint64_t num_pages; + /* syncs main thread and channels */ + QemuSemaphore sem_sync; + /* used for de-compression methods */ + void *data; +} MultiFDRecvParams; + +typedef struct { + /* Setup for sending side */ + int (*send_setup)(MultiFDSendParams *p, Error **errp); + /* Cleanup for sending side */ + void (*send_cleanup)(MultiFDSendParams *p, Error **errp); + /* Prepare the send packet */ + int (*send_prepare)(MultiFDSendParams *p, uint32_t used, Error **errp); + /* Write the send packet */ + int (*send_write)(MultiFDSendParams *p, uint32_t used, Error **errp); + /* Setup for receiving side */ + int (*recv_setup)(MultiFDRecvParams *p, Error **errp); + /* Cleanup for receiving side */ + void (*recv_cleanup)(MultiFDRecvParams *p); + /* Read all pages */ + int (*recv_pages)(MultiFDRecvParams *p, uint32_t used, Error **errp); +} MultiFDMethods; + +void multifd_register_ops(int method, MultiFDMethods *ops); + +#endif + diff --git a/migration/page_cache.c b/migration/page_cache.c new file mode 100644 index 000000000..6d4f7a9bb --- /dev/null +++ b/migration/page_cache.c @@ -0,0 +1,175 @@ +/* + * Page cache for QEMU + * The cache is base on a hash of the page address + * + * Copyright 2012 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Orit Wasserman <owasserm@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +#include "qemu/osdep.h" + +#include "qapi/qmp/qerror.h" +#include "qapi/error.h" +#include "qemu/host-utils.h" +#include "page_cache.h" +#include "trace.h" + +/* the page in cache will not be replaced in two cycles */ +#define CACHED_PAGE_LIFETIME 2 + +typedef struct CacheItem CacheItem; + +struct CacheItem { + uint64_t it_addr; + uint64_t it_age; + uint8_t *it_data; +}; + +struct PageCache { + CacheItem *page_cache; + size_t page_size; + size_t max_num_items; + size_t num_items; +}; + +PageCache *cache_init(uint64_t new_size, size_t page_size, Error **errp) +{ + int64_t i; + size_t num_pages = new_size / page_size; + PageCache *cache; + + if (new_size < page_size) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", + "is smaller than one target page size"); + return NULL; + } + + /* round down to the nearest power of 2 */ + if (!is_power_of_2(num_pages)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", + "is not a power of two number of pages"); + return NULL; + } + + /* We prefer not to abort if there is no memory */ + cache = g_try_malloc(sizeof(*cache)); + if (!cache) { + error_setg(errp, "Failed to allocate cache"); + return NULL; + } + cache->page_size = page_size; + cache->num_items = 0; + cache->max_num_items = num_pages; + + trace_migration_pagecache_init(cache->max_num_items); + + /* We prefer not to abort if there is no memory */ + cache->page_cache = g_try_malloc((cache->max_num_items) * + sizeof(*cache->page_cache)); + if (!cache->page_cache) { + error_setg(errp, "Failed to allocate page cache"); + g_free(cache); + return NULL; + } + + for (i = 0; i < cache->max_num_items; i++) { + cache->page_cache[i].it_data = NULL; + cache->page_cache[i].it_age = 0; + cache->page_cache[i].it_addr = -1; + } + + return cache; +} + +void cache_fini(PageCache *cache) +{ + int64_t i; + + g_assert(cache); + g_assert(cache->page_cache); + + for (i = 0; i < cache->max_num_items; i++) { + g_free(cache->page_cache[i].it_data); + } + + g_free(cache->page_cache); + cache->page_cache = NULL; + g_free(cache); +} + +static size_t cache_get_cache_pos(const PageCache *cache, + uint64_t address) +{ + g_assert(cache->max_num_items); + return (address / cache->page_size) & (cache->max_num_items - 1); +} + +static CacheItem *cache_get_by_addr(const PageCache *cache, uint64_t addr) +{ + size_t pos; + + g_assert(cache); + g_assert(cache->page_cache); + + pos = cache_get_cache_pos(cache, addr); + + return &cache->page_cache[pos]; +} + +uint8_t *get_cached_data(const PageCache *cache, uint64_t addr) +{ + return cache_get_by_addr(cache, addr)->it_data; +} + +bool cache_is_cached(const PageCache *cache, uint64_t addr, + uint64_t current_age) +{ + CacheItem *it; + + it = cache_get_by_addr(cache, addr); + + if (it->it_addr == addr) { + /* update the it_age when the cache hit */ + it->it_age = current_age; + return true; + } + return false; +} + +int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata, + uint64_t current_age) +{ + + CacheItem *it; + + /* actual update of entry */ + it = cache_get_by_addr(cache, addr); + + if (it->it_data && it->it_addr != addr && + it->it_age + CACHED_PAGE_LIFETIME > current_age) { + /* the cache page is fresh, don't replace it */ + return -1; + } + /* allocate page */ + if (!it->it_data) { + it->it_data = g_try_malloc(cache->page_size); + if (!it->it_data) { + trace_migration_pagecache_insert(); + return -1; + } + cache->num_items++; + } + + memcpy(it->it_data, pdata, cache->page_size); + + it->it_age = current_age; + it->it_addr = addr; + + return 0; +} diff --git a/migration/page_cache.h b/migration/page_cache.h new file mode 100644 index 000000000..8733b4df6 --- /dev/null +++ b/migration/page_cache.h @@ -0,0 +1,74 @@ +/* + * Page cache for QEMU + * The cache is base on a hash of the page address + * + * Copyright 2012 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Orit Wasserman <owasserm@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +#ifndef PAGE_CACHE_H +#define PAGE_CACHE_H + +/* Page cache for storing guest pages */ +typedef struct PageCache PageCache; + +/** + * cache_init: Initialize the page cache + * + * + * Returns new allocated cache or NULL on error + * + * @cache_size: cache size in bytes + * @page_size: cache page size + * @errp: set *errp if the check failed, with reason + */ +PageCache *cache_init(uint64_t cache_size, size_t page_size, Error **errp); +/** + * cache_fini: free all cache resources + * @cache pointer to the PageCache struct + */ +void cache_fini(PageCache *cache); + +/** + * cache_is_cached: Checks to see if the page is cached + * + * Returns %true if page is cached + * + * @cache pointer to the PageCache struct + * @addr: page addr + * @current_age: current bitmap generation + */ +bool cache_is_cached(const PageCache *cache, uint64_t addr, + uint64_t current_age); + +/** + * get_cached_data: Get the data cached for an addr + * + * Returns pointer to the data cached or NULL if not cached + * + * @cache pointer to the PageCache struct + * @addr: page addr + */ +uint8_t *get_cached_data(const PageCache *cache, uint64_t addr); + +/** + * cache_insert: insert the page into the cache. the page cache + * will dup the data on insert. the previous value will be overwritten + * + * Returns -1 when the page isn't inserted into cache + * + * @cache pointer to the PageCache struct + * @addr: page address + * @pdata: pointer to the page + * @current_age: current bitmap generation + */ +int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata, + uint64_t current_age); + +#endif diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c new file mode 100644 index 000000000..d18b5d05b --- /dev/null +++ b/migration/postcopy-ram.c @@ -0,0 +1,1471 @@ +/* + * Postcopy migration for RAM + * + * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Dave Gilbert <dgilbert@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +/* + * Postcopy is a migration technique where the execution flips from the + * source to the destination before all the data has been copied. + */ + +#include "qemu/osdep.h" +#include "qemu/rcu.h" +#include "exec/target_page.h" +#include "migration.h" +#include "qemu-file.h" +#include "savevm.h" +#include "postcopy-ram.h" +#include "ram.h" +#include "qapi/error.h" +#include "qemu/notify.h" +#include "qemu/rcu.h" +#include "sysemu/sysemu.h" +#include "qemu/error-report.h" +#include "trace.h" +#include "hw/boards.h" +#include "exec/ramblock.h" + +/* Arbitrary limit on size of each discard command, + * keeps them around ~200 bytes + */ +#define MAX_DISCARDS_PER_COMMAND 12 + +struct PostcopyDiscardState { + const char *ramblock_name; + uint16_t cur_entry; + /* + * Start and length of a discard range (bytes) + */ + uint64_t start_list[MAX_DISCARDS_PER_COMMAND]; + uint64_t length_list[MAX_DISCARDS_PER_COMMAND]; + unsigned int nsentwords; + unsigned int nsentcmds; +}; + +static NotifierWithReturnList postcopy_notifier_list; + +void postcopy_infrastructure_init(void) +{ + notifier_with_return_list_init(&postcopy_notifier_list); +} + +void postcopy_add_notifier(NotifierWithReturn *nn) +{ + notifier_with_return_list_add(&postcopy_notifier_list, nn); +} + +void postcopy_remove_notifier(NotifierWithReturn *n) +{ + notifier_with_return_remove(n); +} + +int postcopy_notify(enum PostcopyNotifyReason reason, Error **errp) +{ + struct PostcopyNotifyData pnd; + pnd.reason = reason; + pnd.errp = errp; + + return notifier_with_return_list_notify(&postcopy_notifier_list, + &pnd); +} + +/* Postcopy needs to detect accesses to pages that haven't yet been copied + * across, and efficiently map new pages in, the techniques for doing this + * are target OS specific. + */ +#if defined(__linux__) + +#include <poll.h> +#include <sys/ioctl.h> +#include <sys/syscall.h> +#include <asm/types.h> /* for __u64 */ +#endif + +#if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD) +#include <sys/eventfd.h> +#include <linux/userfaultfd.h> + +typedef struct PostcopyBlocktimeContext { + /* time when page fault initiated per vCPU */ + uint32_t *page_fault_vcpu_time; + /* page address per vCPU */ + uintptr_t *vcpu_addr; + uint32_t total_blocktime; + /* blocktime per vCPU */ + uint32_t *vcpu_blocktime; + /* point in time when last page fault was initiated */ + uint32_t last_begin; + /* number of vCPU are suspended */ + int smp_cpus_down; + uint64_t start_time; + + /* + * Handler for exit event, necessary for + * releasing whole blocktime_ctx + */ + Notifier exit_notifier; +} PostcopyBlocktimeContext; + +static void destroy_blocktime_context(struct PostcopyBlocktimeContext *ctx) +{ + g_free(ctx->page_fault_vcpu_time); + g_free(ctx->vcpu_addr); + g_free(ctx->vcpu_blocktime); + g_free(ctx); +} + +static void migration_exit_cb(Notifier *n, void *data) +{ + PostcopyBlocktimeContext *ctx = container_of(n, PostcopyBlocktimeContext, + exit_notifier); + destroy_blocktime_context(ctx); +} + +static struct PostcopyBlocktimeContext *blocktime_context_new(void) +{ + MachineState *ms = MACHINE(qdev_get_machine()); + unsigned int smp_cpus = ms->smp.cpus; + PostcopyBlocktimeContext *ctx = g_new0(PostcopyBlocktimeContext, 1); + ctx->page_fault_vcpu_time = g_new0(uint32_t, smp_cpus); + ctx->vcpu_addr = g_new0(uintptr_t, smp_cpus); + ctx->vcpu_blocktime = g_new0(uint32_t, smp_cpus); + + ctx->exit_notifier.notify = migration_exit_cb; + ctx->start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + qemu_add_exit_notifier(&ctx->exit_notifier); + return ctx; +} + +static uint32List *get_vcpu_blocktime_list(PostcopyBlocktimeContext *ctx) +{ + MachineState *ms = MACHINE(qdev_get_machine()); + uint32List *list = NULL; + int i; + + for (i = ms->smp.cpus - 1; i >= 0; i--) { + QAPI_LIST_PREPEND(list, ctx->vcpu_blocktime[i]); + } + + return list; +} + +/* + * This function just populates MigrationInfo from postcopy's + * blocktime context. It will not populate MigrationInfo, + * unless postcopy-blocktime capability was set. + * + * @info: pointer to MigrationInfo to populate + */ +void fill_destination_postcopy_migration_info(MigrationInfo *info) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + PostcopyBlocktimeContext *bc = mis->blocktime_ctx; + + if (!bc) { + return; + } + + info->has_postcopy_blocktime = true; + info->postcopy_blocktime = bc->total_blocktime; + info->has_postcopy_vcpu_blocktime = true; + info->postcopy_vcpu_blocktime = get_vcpu_blocktime_list(bc); +} + +static uint32_t get_postcopy_total_blocktime(void) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + PostcopyBlocktimeContext *bc = mis->blocktime_ctx; + + if (!bc) { + return 0; + } + + return bc->total_blocktime; +} + +/** + * receive_ufd_features: check userfault fd features, to request only supported + * features in the future. + * + * Returns: true on success + * + * __NR_userfaultfd - should be checked before + * @features: out parameter will contain uffdio_api.features provided by kernel + * in case of success + */ +static bool receive_ufd_features(uint64_t *features) +{ + struct uffdio_api api_struct = {0}; + int ufd; + bool ret = true; + + /* if we are here __NR_userfaultfd should exists */ + ufd = syscall(__NR_userfaultfd, O_CLOEXEC); + if (ufd == -1) { + error_report("%s: syscall __NR_userfaultfd failed: %s", __func__, + strerror(errno)); + return false; + } + + /* ask features */ + api_struct.api = UFFD_API; + api_struct.features = 0; + if (ioctl(ufd, UFFDIO_API, &api_struct)) { + error_report("%s: UFFDIO_API failed: %s", __func__, + strerror(errno)); + ret = false; + goto release_ufd; + } + + *features = api_struct.features; + +release_ufd: + close(ufd); + return ret; +} + +/** + * request_ufd_features: this function should be called only once on a newly + * opened ufd, subsequent calls will lead to error. + * + * Returns: true on success + * + * @ufd: fd obtained from userfaultfd syscall + * @features: bit mask see UFFD_API_FEATURES + */ +static bool request_ufd_features(int ufd, uint64_t features) +{ + struct uffdio_api api_struct = {0}; + uint64_t ioctl_mask; + + api_struct.api = UFFD_API; + api_struct.features = features; + if (ioctl(ufd, UFFDIO_API, &api_struct)) { + error_report("%s failed: UFFDIO_API failed: %s", __func__, + strerror(errno)); + return false; + } + + ioctl_mask = (__u64)1 << _UFFDIO_REGISTER | + (__u64)1 << _UFFDIO_UNREGISTER; + if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) { + error_report("Missing userfault features: %" PRIx64, + (uint64_t)(~api_struct.ioctls & ioctl_mask)); + return false; + } + + return true; +} + +static bool ufd_check_and_apply(int ufd, MigrationIncomingState *mis) +{ + uint64_t asked_features = 0; + static uint64_t supported_features; + + /* + * it's not possible to + * request UFFD_API twice per one fd + * userfault fd features is persistent + */ + if (!supported_features) { + if (!receive_ufd_features(&supported_features)) { + error_report("%s failed", __func__); + return false; + } + } + +#ifdef UFFD_FEATURE_THREAD_ID + if (migrate_postcopy_blocktime() && mis && + UFFD_FEATURE_THREAD_ID & supported_features) { + /* kernel supports that feature */ + /* don't create blocktime_context if it exists */ + if (!mis->blocktime_ctx) { + mis->blocktime_ctx = blocktime_context_new(); + } + + asked_features |= UFFD_FEATURE_THREAD_ID; + } +#endif + + /* + * request features, even if asked_features is 0, due to + * kernel expects UFFD_API before UFFDIO_REGISTER, per + * userfault file descriptor + */ + if (!request_ufd_features(ufd, asked_features)) { + error_report("%s failed: features %" PRIu64, __func__, + asked_features); + return false; + } + + if (qemu_real_host_page_size != ram_pagesize_summary()) { + bool have_hp = false; + /* We've got a huge page */ +#ifdef UFFD_FEATURE_MISSING_HUGETLBFS + have_hp = supported_features & UFFD_FEATURE_MISSING_HUGETLBFS; +#endif + if (!have_hp) { + error_report("Userfault on this host does not support huge pages"); + return false; + } + } + return true; +} + +/* Callback from postcopy_ram_supported_by_host block iterator. + */ +static int test_ramblock_postcopiable(RAMBlock *rb, void *opaque) +{ + const char *block_name = qemu_ram_get_idstr(rb); + ram_addr_t length = qemu_ram_get_used_length(rb); + size_t pagesize = qemu_ram_pagesize(rb); + + if (length % pagesize) { + error_report("Postcopy requires RAM blocks to be a page size multiple," + " block %s is 0x" RAM_ADDR_FMT " bytes with a " + "page size of 0x%zx", block_name, length, pagesize); + return 1; + } + return 0; +} + +/* + * Note: This has the side effect of munlock'ing all of RAM, that's + * normally fine since if the postcopy succeeds it gets turned back on at the + * end. + */ +bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) +{ + long pagesize = qemu_real_host_page_size; + int ufd = -1; + bool ret = false; /* Error unless we change it */ + void *testarea = NULL; + struct uffdio_register reg_struct; + struct uffdio_range range_struct; + uint64_t feature_mask; + Error *local_err = NULL; + + if (qemu_target_page_size() > pagesize) { + error_report("Target page size bigger than host page size"); + goto out; + } + + ufd = syscall(__NR_userfaultfd, O_CLOEXEC); + if (ufd == -1) { + error_report("%s: userfaultfd not available: %s", __func__, + strerror(errno)); + goto out; + } + + /* Give devices a chance to object */ + if (postcopy_notify(POSTCOPY_NOTIFY_PROBE, &local_err)) { + error_report_err(local_err); + goto out; + } + + /* Version and features check */ + if (!ufd_check_and_apply(ufd, mis)) { + goto out; + } + + /* We don't support postcopy with shared RAM yet */ + if (foreach_not_ignored_block(test_ramblock_postcopiable, NULL)) { + goto out; + } + + /* + * userfault and mlock don't go together; we'll put it back later if + * it was enabled. + */ + if (munlockall()) { + error_report("%s: munlockall: %s", __func__, strerror(errno)); + goto out; + } + + /* + * We need to check that the ops we need are supported on anon memory + * To do that we need to register a chunk and see the flags that + * are returned. + */ + testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | + MAP_ANONYMOUS, -1, 0); + if (testarea == MAP_FAILED) { + error_report("%s: Failed to map test area: %s", __func__, + strerror(errno)); + goto out; + } + g_assert(QEMU_PTR_IS_ALIGNED(testarea, pagesize)); + + reg_struct.range.start = (uintptr_t)testarea; + reg_struct.range.len = pagesize; + reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; + + if (ioctl(ufd, UFFDIO_REGISTER, ®_struct)) { + error_report("%s userfault register: %s", __func__, strerror(errno)); + goto out; + } + + range_struct.start = (uintptr_t)testarea; + range_struct.len = pagesize; + if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) { + error_report("%s userfault unregister: %s", __func__, strerror(errno)); + goto out; + } + + feature_mask = (__u64)1 << _UFFDIO_WAKE | + (__u64)1 << _UFFDIO_COPY | + (__u64)1 << _UFFDIO_ZEROPAGE; + if ((reg_struct.ioctls & feature_mask) != feature_mask) { + error_report("Missing userfault map features: %" PRIx64, + (uint64_t)(~reg_struct.ioctls & feature_mask)); + goto out; + } + + /* Success! */ + ret = true; +out: + if (testarea) { + munmap(testarea, pagesize); + } + if (ufd != -1) { + close(ufd); + } + return ret; +} + +/* + * Setup an area of RAM so that it *can* be used for postcopy later; this + * must be done right at the start prior to pre-copy. + * opaque should be the MIS. + */ +static int init_range(RAMBlock *rb, void *opaque) +{ + const char *block_name = qemu_ram_get_idstr(rb); + void *host_addr = qemu_ram_get_host_addr(rb); + ram_addr_t offset = qemu_ram_get_offset(rb); + ram_addr_t length = qemu_ram_get_used_length(rb); + trace_postcopy_init_range(block_name, host_addr, offset, length); + + /* + * Save the used_length before running the guest. In case we have to + * resize RAM blocks when syncing RAM block sizes from the source during + * precopy, we'll update it manually via the ram block notifier. + */ + rb->postcopy_length = length; + + /* + * We need the whole of RAM to be truly empty for postcopy, so things + * like ROMs and any data tables built during init must be zero'd + * - we're going to get the copy from the source anyway. + * (Precopy will just overwrite this data, so doesn't need the discard) + */ + if (ram_discard_range(block_name, 0, length)) { + return -1; + } + + return 0; +} + +/* + * At the end of migration, undo the effects of init_range + * opaque should be the MIS. + */ +static int cleanup_range(RAMBlock *rb, void *opaque) +{ + const char *block_name = qemu_ram_get_idstr(rb); + void *host_addr = qemu_ram_get_host_addr(rb); + ram_addr_t offset = qemu_ram_get_offset(rb); + ram_addr_t length = rb->postcopy_length; + MigrationIncomingState *mis = opaque; + struct uffdio_range range_struct; + trace_postcopy_cleanup_range(block_name, host_addr, offset, length); + + /* + * We turned off hugepage for the precopy stage with postcopy enabled + * we can turn it back on now. + */ + qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE); + + /* + * We can also turn off userfault now since we should have all the + * pages. It can be useful to leave it on to debug postcopy + * if you're not sure it's always getting every page. + */ + range_struct.start = (uintptr_t)host_addr; + range_struct.len = length; + + if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) { + error_report("%s: userfault unregister %s", __func__, strerror(errno)); + + return -1; + } + + return 0; +} + +/* + * Initialise postcopy-ram, setting the RAM to a state where we can go into + * postcopy later; must be called prior to any precopy. + * called from arch_init's similarly named ram_postcopy_incoming_init + */ +int postcopy_ram_incoming_init(MigrationIncomingState *mis) +{ + if (foreach_not_ignored_block(init_range, NULL)) { + return -1; + } + + return 0; +} + +/* + * At the end of a migration where postcopy_ram_incoming_init was called. + */ +int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) +{ + trace_postcopy_ram_incoming_cleanup_entry(); + + if (mis->have_fault_thread) { + Error *local_err = NULL; + + /* Let the fault thread quit */ + qatomic_set(&mis->fault_thread_quit, 1); + postcopy_fault_thread_notify(mis); + trace_postcopy_ram_incoming_cleanup_join(); + qemu_thread_join(&mis->fault_thread); + + if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_END, &local_err)) { + error_report_err(local_err); + return -1; + } + + if (foreach_not_ignored_block(cleanup_range, mis)) { + return -1; + } + + trace_postcopy_ram_incoming_cleanup_closeuf(); + close(mis->userfault_fd); + close(mis->userfault_event_fd); + mis->have_fault_thread = false; + } + + if (enable_mlock) { + if (os_mlock() < 0) { + error_report("mlock: %s", strerror(errno)); + /* + * It doesn't feel right to fail at this point, we have a valid + * VM state. + */ + } + } + + if (mis->postcopy_tmp_page) { + munmap(mis->postcopy_tmp_page, mis->largest_page_size); + mis->postcopy_tmp_page = NULL; + } + if (mis->postcopy_tmp_zero_page) { + munmap(mis->postcopy_tmp_zero_page, mis->largest_page_size); + mis->postcopy_tmp_zero_page = NULL; + } + trace_postcopy_ram_incoming_cleanup_blocktime( + get_postcopy_total_blocktime()); + + trace_postcopy_ram_incoming_cleanup_exit(); + return 0; +} + +/* + * Disable huge pages on an area + */ +static int nhp_range(RAMBlock *rb, void *opaque) +{ + const char *block_name = qemu_ram_get_idstr(rb); + void *host_addr = qemu_ram_get_host_addr(rb); + ram_addr_t offset = qemu_ram_get_offset(rb); + ram_addr_t length = rb->postcopy_length; + trace_postcopy_nhp_range(block_name, host_addr, offset, length); + + /* + * Before we do discards we need to ensure those discards really + * do delete areas of the page, even if THP thinks a hugepage would + * be a good idea, so force hugepages off. + */ + qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE); + + return 0; +} + +/* + * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard + * however leaving it until after precopy means that most of the precopy + * data is still THPd + */ +int postcopy_ram_prepare_discard(MigrationIncomingState *mis) +{ + if (foreach_not_ignored_block(nhp_range, mis)) { + return -1; + } + + postcopy_state_set(POSTCOPY_INCOMING_DISCARD); + + return 0; +} + +/* + * Mark the given area of RAM as requiring notification to unwritten areas + * Used as a callback on foreach_not_ignored_block. + * host_addr: Base of area to mark + * offset: Offset in the whole ram arena + * length: Length of the section + * opaque: MigrationIncomingState pointer + * Returns 0 on success + */ +static int ram_block_enable_notify(RAMBlock *rb, void *opaque) +{ + MigrationIncomingState *mis = opaque; + struct uffdio_register reg_struct; + + reg_struct.range.start = (uintptr_t)qemu_ram_get_host_addr(rb); + reg_struct.range.len = rb->postcopy_length; + reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; + + /* Now tell our userfault_fd that it's responsible for this area */ + if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, ®_struct)) { + error_report("%s userfault register: %s", __func__, strerror(errno)); + return -1; + } + if (!(reg_struct.ioctls & ((__u64)1 << _UFFDIO_COPY))) { + error_report("%s userfault: Region doesn't support COPY", __func__); + return -1; + } + if (reg_struct.ioctls & ((__u64)1 << _UFFDIO_ZEROPAGE)) { + qemu_ram_set_uf_zeroable(rb); + } + + return 0; +} + +int postcopy_wake_shared(struct PostCopyFD *pcfd, + uint64_t client_addr, + RAMBlock *rb) +{ + size_t pagesize = qemu_ram_pagesize(rb); + struct uffdio_range range; + int ret; + trace_postcopy_wake_shared(client_addr, qemu_ram_get_idstr(rb)); + range.start = ROUND_DOWN(client_addr, pagesize); + range.len = pagesize; + ret = ioctl(pcfd->fd, UFFDIO_WAKE, &range); + if (ret) { + error_report("%s: Failed to wake: %zx in %s (%s)", + __func__, (size_t)client_addr, qemu_ram_get_idstr(rb), + strerror(errno)); + } + return ret; +} + +static int postcopy_request_page(MigrationIncomingState *mis, RAMBlock *rb, + ram_addr_t start, uint64_t haddr) +{ + void *aligned = (void *)(uintptr_t)ROUND_DOWN(haddr, qemu_ram_pagesize(rb)); + + /* + * Discarded pages (via RamDiscardManager) are never migrated. On unlikely + * access, place a zeropage, which will also set the relevant bits in the + * recv_bitmap accordingly, so we won't try placing a zeropage twice. + * + * Checking a single bit is sufficient to handle pagesize > TPS as either + * all relevant bits are set or not. + */ + assert(QEMU_IS_ALIGNED(start, qemu_ram_pagesize(rb))); + if (ramblock_page_is_discarded(rb, start)) { + bool received = ramblock_recv_bitmap_test_byte_offset(rb, start); + + return received ? 0 : postcopy_place_page_zero(mis, aligned, rb); + } + + return migrate_send_rp_req_pages(mis, rb, start, haddr); +} + +/* + * Callback from shared fault handlers to ask for a page, + * the page must be specified by a RAMBlock and an offset in that rb + * Note: Only for use by shared fault handlers (in fault thread) + */ +int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, + uint64_t client_addr, uint64_t rb_offset) +{ + uint64_t aligned_rbo = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb)); + MigrationIncomingState *mis = migration_incoming_get_current(); + + trace_postcopy_request_shared_page(pcfd->idstr, qemu_ram_get_idstr(rb), + rb_offset); + if (ramblock_recv_bitmap_test_byte_offset(rb, aligned_rbo)) { + trace_postcopy_request_shared_page_present(pcfd->idstr, + qemu_ram_get_idstr(rb), rb_offset); + return postcopy_wake_shared(pcfd, client_addr, rb); + } + postcopy_request_page(mis, rb, aligned_rbo, client_addr); + return 0; +} + +static int get_mem_fault_cpu_index(uint32_t pid) +{ + CPUState *cpu_iter; + + CPU_FOREACH(cpu_iter) { + if (cpu_iter->thread_id == pid) { + trace_get_mem_fault_cpu_index(cpu_iter->cpu_index, pid); + return cpu_iter->cpu_index; + } + } + trace_get_mem_fault_cpu_index(-1, pid); + return -1; +} + +static uint32_t get_low_time_offset(PostcopyBlocktimeContext *dc) +{ + int64_t start_time_offset = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - + dc->start_time; + return start_time_offset < 1 ? 1 : start_time_offset & UINT32_MAX; +} + +/* + * This function is being called when pagefault occurs. It + * tracks down vCPU blocking time. + * + * @addr: faulted host virtual address + * @ptid: faulted process thread id + * @rb: ramblock appropriate to addr + */ +static void mark_postcopy_blocktime_begin(uintptr_t addr, uint32_t ptid, + RAMBlock *rb) +{ + int cpu, already_received; + MigrationIncomingState *mis = migration_incoming_get_current(); + PostcopyBlocktimeContext *dc = mis->blocktime_ctx; + uint32_t low_time_offset; + + if (!dc || ptid == 0) { + return; + } + cpu = get_mem_fault_cpu_index(ptid); + if (cpu < 0) { + return; + } + + low_time_offset = get_low_time_offset(dc); + if (dc->vcpu_addr[cpu] == 0) { + qatomic_inc(&dc->smp_cpus_down); + } + + qatomic_xchg(&dc->last_begin, low_time_offset); + qatomic_xchg(&dc->page_fault_vcpu_time[cpu], low_time_offset); + qatomic_xchg(&dc->vcpu_addr[cpu], addr); + + /* + * check it here, not at the beginning of the function, + * due to, check could occur early than bitmap_set in + * qemu_ufd_copy_ioctl + */ + already_received = ramblock_recv_bitmap_test(rb, (void *)addr); + if (already_received) { + qatomic_xchg(&dc->vcpu_addr[cpu], 0); + qatomic_xchg(&dc->page_fault_vcpu_time[cpu], 0); + qatomic_dec(&dc->smp_cpus_down); + } + trace_mark_postcopy_blocktime_begin(addr, dc, dc->page_fault_vcpu_time[cpu], + cpu, already_received); +} + +/* + * This function just provide calculated blocktime per cpu and trace it. + * Total blocktime is calculated in mark_postcopy_blocktime_end. + * + * + * Assume we have 3 CPU + * + * S1 E1 S1 E1 + * -----***********------------xxx***************------------------------> CPU1 + * + * S2 E2 + * ------------****************xxx---------------------------------------> CPU2 + * + * S3 E3 + * ------------------------****xxx********-------------------------------> CPU3 + * + * We have sequence S1,S2,E1,S3,S1,E2,E3,E1 + * S2,E1 - doesn't match condition due to sequence S1,S2,E1 doesn't include CPU3 + * S3,S1,E2 - sequence includes all CPUs, in this case overlap will be S1,E2 - + * it's a part of total blocktime. + * S1 - here is last_begin + * Legend of the picture is following: + * * - means blocktime per vCPU + * x - means overlapped blocktime (total blocktime) + * + * @addr: host virtual address + */ +static void mark_postcopy_blocktime_end(uintptr_t addr) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + PostcopyBlocktimeContext *dc = mis->blocktime_ctx; + MachineState *ms = MACHINE(qdev_get_machine()); + unsigned int smp_cpus = ms->smp.cpus; + int i, affected_cpu = 0; + bool vcpu_total_blocktime = false; + uint32_t read_vcpu_time, low_time_offset; + + if (!dc) { + return; + } + + low_time_offset = get_low_time_offset(dc); + /* lookup cpu, to clear it, + * that algorithm looks straightforward, but it's not + * optimal, more optimal algorithm is keeping tree or hash + * where key is address value is a list of */ + for (i = 0; i < smp_cpus; i++) { + uint32_t vcpu_blocktime = 0; + + read_vcpu_time = qatomic_fetch_add(&dc->page_fault_vcpu_time[i], 0); + if (qatomic_fetch_add(&dc->vcpu_addr[i], 0) != addr || + read_vcpu_time == 0) { + continue; + } + qatomic_xchg(&dc->vcpu_addr[i], 0); + vcpu_blocktime = low_time_offset - read_vcpu_time; + affected_cpu += 1; + /* we need to know is that mark_postcopy_end was due to + * faulted page, another possible case it's prefetched + * page and in that case we shouldn't be here */ + if (!vcpu_total_blocktime && + qatomic_fetch_add(&dc->smp_cpus_down, 0) == smp_cpus) { + vcpu_total_blocktime = true; + } + /* continue cycle, due to one page could affect several vCPUs */ + dc->vcpu_blocktime[i] += vcpu_blocktime; + } + + qatomic_sub(&dc->smp_cpus_down, affected_cpu); + if (vcpu_total_blocktime) { + dc->total_blocktime += low_time_offset - qatomic_fetch_add( + &dc->last_begin, 0); + } + trace_mark_postcopy_blocktime_end(addr, dc, dc->total_blocktime, + affected_cpu); +} + +static bool postcopy_pause_fault_thread(MigrationIncomingState *mis) +{ + trace_postcopy_pause_fault_thread(); + + qemu_sem_wait(&mis->postcopy_pause_sem_fault); + + trace_postcopy_pause_fault_thread_continued(); + + return true; +} + +/* + * Handle faults detected by the USERFAULT markings + */ +static void *postcopy_ram_fault_thread(void *opaque) +{ + MigrationIncomingState *mis = opaque; + struct uffd_msg msg; + int ret; + size_t index; + RAMBlock *rb = NULL; + + trace_postcopy_ram_fault_thread_entry(); + rcu_register_thread(); + mis->last_rb = NULL; /* last RAMBlock we sent part of */ + qemu_sem_post(&mis->fault_thread_sem); + + struct pollfd *pfd; + size_t pfd_len = 2 + mis->postcopy_remote_fds->len; + + pfd = g_new0(struct pollfd, pfd_len); + + pfd[0].fd = mis->userfault_fd; + pfd[0].events = POLLIN; + pfd[1].fd = mis->userfault_event_fd; + pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */ + trace_postcopy_ram_fault_thread_fds_core(pfd[0].fd, pfd[1].fd); + for (index = 0; index < mis->postcopy_remote_fds->len; index++) { + struct PostCopyFD *pcfd = &g_array_index(mis->postcopy_remote_fds, + struct PostCopyFD, index); + pfd[2 + index].fd = pcfd->fd; + pfd[2 + index].events = POLLIN; + trace_postcopy_ram_fault_thread_fds_extra(2 + index, pcfd->idstr, + pcfd->fd); + } + + while (true) { + ram_addr_t rb_offset; + int poll_result; + + /* + * We're mainly waiting for the kernel to give us a faulting HVA, + * however we can be told to quit via userfault_quit_fd which is + * an eventfd + */ + + poll_result = poll(pfd, pfd_len, -1 /* Wait forever */); + if (poll_result == -1) { + error_report("%s: userfault poll: %s", __func__, strerror(errno)); + break; + } + + if (!mis->to_src_file) { + /* + * Possibly someone tells us that the return path is + * broken already using the event. We should hold until + * the channel is rebuilt. + */ + if (postcopy_pause_fault_thread(mis)) { + /* Continue to read the userfaultfd */ + } else { + error_report("%s: paused but don't allow to continue", + __func__); + break; + } + } + + if (pfd[1].revents) { + uint64_t tmp64 = 0; + + /* Consume the signal */ + if (read(mis->userfault_event_fd, &tmp64, 8) != 8) { + /* Nothing obviously nicer than posting this error. */ + error_report("%s: read() failed", __func__); + } + + if (qatomic_read(&mis->fault_thread_quit)) { + trace_postcopy_ram_fault_thread_quit(); + break; + } + } + + if (pfd[0].revents) { + poll_result--; + ret = read(mis->userfault_fd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) { + if (errno == EAGAIN) { + /* + * if a wake up happens on the other thread just after + * the poll, there is nothing to read. + */ + continue; + } + if (ret < 0) { + error_report("%s: Failed to read full userfault " + "message: %s", + __func__, strerror(errno)); + break; + } else { + error_report("%s: Read %d bytes from userfaultfd " + "expected %zd", + __func__, ret, sizeof(msg)); + break; /* Lost alignment, don't know what we'd read next */ + } + } + if (msg.event != UFFD_EVENT_PAGEFAULT) { + error_report("%s: Read unexpected event %ud from userfaultfd", + __func__, msg.event); + continue; /* It's not a page fault, shouldn't happen */ + } + + rb = qemu_ram_block_from_host( + (void *)(uintptr_t)msg.arg.pagefault.address, + true, &rb_offset); + if (!rb) { + error_report("postcopy_ram_fault_thread: Fault outside guest: %" + PRIx64, (uint64_t)msg.arg.pagefault.address); + break; + } + + rb_offset = ROUND_DOWN(rb_offset, qemu_ram_pagesize(rb)); + trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address, + qemu_ram_get_idstr(rb), + rb_offset, + msg.arg.pagefault.feat.ptid); + mark_postcopy_blocktime_begin( + (uintptr_t)(msg.arg.pagefault.address), + msg.arg.pagefault.feat.ptid, rb); + +retry: + /* + * Send the request to the source - we want to request one + * of our host page sizes (which is >= TPS) + */ + ret = postcopy_request_page(mis, rb, rb_offset, + msg.arg.pagefault.address); + if (ret) { + /* May be network failure, try to wait for recovery */ + if (ret == -EIO && postcopy_pause_fault_thread(mis)) { + /* We got reconnected somehow, try to continue */ + goto retry; + } else { + /* This is a unavoidable fault */ + error_report("%s: postcopy_request_page() get %d", + __func__, ret); + break; + } + } + } + + /* Now handle any requests from external processes on shared memory */ + /* TODO: May need to handle devices deregistering during postcopy */ + for (index = 2; index < pfd_len && poll_result; index++) { + if (pfd[index].revents) { + struct PostCopyFD *pcfd = + &g_array_index(mis->postcopy_remote_fds, + struct PostCopyFD, index - 2); + + poll_result--; + if (pfd[index].revents & POLLERR) { + error_report("%s: POLLERR on poll %zd fd=%d", + __func__, index, pcfd->fd); + pfd[index].events = 0; + continue; + } + + ret = read(pcfd->fd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) { + if (errno == EAGAIN) { + /* + * if a wake up happens on the other thread just after + * the poll, there is nothing to read. + */ + continue; + } + if (ret < 0) { + error_report("%s: Failed to read full userfault " + "message: %s (shared) revents=%d", + __func__, strerror(errno), + pfd[index].revents); + /*TODO: Could just disable this sharer */ + break; + } else { + error_report("%s: Read %d bytes from userfaultfd " + "expected %zd (shared)", + __func__, ret, sizeof(msg)); + /*TODO: Could just disable this sharer */ + break; /*Lost alignment,don't know what we'd read next*/ + } + } + if (msg.event != UFFD_EVENT_PAGEFAULT) { + error_report("%s: Read unexpected event %ud " + "from userfaultfd (shared)", + __func__, msg.event); + continue; /* It's not a page fault, shouldn't happen */ + } + /* Call the device handler registered with us */ + ret = pcfd->handler(pcfd, &msg); + if (ret) { + error_report("%s: Failed to resolve shared fault on %zd/%s", + __func__, index, pcfd->idstr); + /* TODO: Fail? Disable this sharer? */ + } + } + } + } + rcu_unregister_thread(); + trace_postcopy_ram_fault_thread_exit(); + g_free(pfd); + return NULL; +} + +int postcopy_ram_incoming_setup(MigrationIncomingState *mis) +{ + /* Open the fd for the kernel to give us userfaults */ + mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (mis->userfault_fd == -1) { + error_report("%s: Failed to open userfault fd: %s", __func__, + strerror(errno)); + return -1; + } + + /* + * Although the host check already tested the API, we need to + * do the check again as an ABI handshake on the new fd. + */ + if (!ufd_check_and_apply(mis->userfault_fd, mis)) { + return -1; + } + + /* Now an eventfd we use to tell the fault-thread to quit */ + mis->userfault_event_fd = eventfd(0, EFD_CLOEXEC); + if (mis->userfault_event_fd == -1) { + error_report("%s: Opening userfault_event_fd: %s", __func__, + strerror(errno)); + close(mis->userfault_fd); + return -1; + } + + qemu_sem_init(&mis->fault_thread_sem, 0); + qemu_thread_create(&mis->fault_thread, "postcopy/fault", + postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE); + qemu_sem_wait(&mis->fault_thread_sem); + qemu_sem_destroy(&mis->fault_thread_sem); + mis->have_fault_thread = true; + + /* Mark so that we get notified of accesses to unwritten areas */ + if (foreach_not_ignored_block(ram_block_enable_notify, mis)) { + error_report("ram_block_enable_notify failed"); + return -1; + } + + mis->postcopy_tmp_page = mmap(NULL, mis->largest_page_size, + PROT_READ | PROT_WRITE, MAP_PRIVATE | + MAP_ANONYMOUS, -1, 0); + if (mis->postcopy_tmp_page == MAP_FAILED) { + mis->postcopy_tmp_page = NULL; + error_report("%s: Failed to map postcopy_tmp_page %s", + __func__, strerror(errno)); + return -1; + } + + /* + * Map large zero page when kernel can't use UFFDIO_ZEROPAGE for hugepages + */ + mis->postcopy_tmp_zero_page = mmap(NULL, mis->largest_page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (mis->postcopy_tmp_zero_page == MAP_FAILED) { + int e = errno; + mis->postcopy_tmp_zero_page = NULL; + error_report("%s: Failed to map large zero page %s", + __func__, strerror(e)); + return -e; + } + memset(mis->postcopy_tmp_zero_page, '\0', mis->largest_page_size); + + trace_postcopy_ram_enable_notify(); + + return 0; +} + +static int qemu_ufd_copy_ioctl(MigrationIncomingState *mis, void *host_addr, + void *from_addr, uint64_t pagesize, RAMBlock *rb) +{ + int userfault_fd = mis->userfault_fd; + int ret; + + if (from_addr) { + struct uffdio_copy copy_struct; + copy_struct.dst = (uint64_t)(uintptr_t)host_addr; + copy_struct.src = (uint64_t)(uintptr_t)from_addr; + copy_struct.len = pagesize; + copy_struct.mode = 0; + ret = ioctl(userfault_fd, UFFDIO_COPY, ©_struct); + } else { + struct uffdio_zeropage zero_struct; + zero_struct.range.start = (uint64_t)(uintptr_t)host_addr; + zero_struct.range.len = pagesize; + zero_struct.mode = 0; + ret = ioctl(userfault_fd, UFFDIO_ZEROPAGE, &zero_struct); + } + if (!ret) { + qemu_mutex_lock(&mis->page_request_mutex); + ramblock_recv_bitmap_set_range(rb, host_addr, + pagesize / qemu_target_page_size()); + /* + * If this page resolves a page fault for a previous recorded faulted + * address, take a special note to maintain the requested page list. + */ + if (g_tree_lookup(mis->page_requested, host_addr)) { + g_tree_remove(mis->page_requested, host_addr); + mis->page_requested_count--; + trace_postcopy_page_req_del(host_addr, mis->page_requested_count); + } + qemu_mutex_unlock(&mis->page_request_mutex); + mark_postcopy_blocktime_end((uintptr_t)host_addr); + } + return ret; +} + +int postcopy_notify_shared_wake(RAMBlock *rb, uint64_t offset) +{ + int i; + MigrationIncomingState *mis = migration_incoming_get_current(); + GArray *pcrfds = mis->postcopy_remote_fds; + + for (i = 0; i < pcrfds->len; i++) { + struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); + int ret = cur->waker(cur, rb, offset); + if (ret) { + return ret; + } + } + return 0; +} + +/* + * Place a host page (from) at (host) atomically + * returns 0 on success + */ +int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, + RAMBlock *rb) +{ + size_t pagesize = qemu_ram_pagesize(rb); + + /* copy also acks to the kernel waking the stalled thread up + * TODO: We can inhibit that ack and only do it if it was requested + * which would be slightly cheaper, but we'd have to be careful + * of the order of updating our page state. + */ + if (qemu_ufd_copy_ioctl(mis, host, from, pagesize, rb)) { + int e = errno; + error_report("%s: %s copy host: %p from: %p (size: %zd)", + __func__, strerror(e), host, from, pagesize); + + return -e; + } + + trace_postcopy_place_page(host); + return postcopy_notify_shared_wake(rb, + qemu_ram_block_host_offset(rb, host)); +} + +/* + * Place a zero page at (host) atomically + * returns 0 on success + */ +int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, + RAMBlock *rb) +{ + size_t pagesize = qemu_ram_pagesize(rb); + trace_postcopy_place_page_zero(host); + + /* Normal RAMBlocks can zero a page using UFFDIO_ZEROPAGE + * but it's not available for everything (e.g. hugetlbpages) + */ + if (qemu_ram_is_uf_zeroable(rb)) { + if (qemu_ufd_copy_ioctl(mis, host, NULL, pagesize, rb)) { + int e = errno; + error_report("%s: %s zero host: %p", + __func__, strerror(e), host); + + return -e; + } + return postcopy_notify_shared_wake(rb, + qemu_ram_block_host_offset(rb, + host)); + } else { + return postcopy_place_page(mis, host, mis->postcopy_tmp_zero_page, rb); + } +} + +#else +/* No target OS support, stubs just fail */ +void fill_destination_postcopy_migration_info(MigrationInfo *info) +{ +} + +bool postcopy_ram_supported_by_host(MigrationIncomingState *mis) +{ + error_report("%s: No OS support", __func__); + return false; +} + +int postcopy_ram_incoming_init(MigrationIncomingState *mis) +{ + error_report("postcopy_ram_incoming_init: No OS support"); + return -1; +} + +int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) +{ + assert(0); + return -1; +} + +int postcopy_ram_prepare_discard(MigrationIncomingState *mis) +{ + assert(0); + return -1; +} + +int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, + uint64_t client_addr, uint64_t rb_offset) +{ + assert(0); + return -1; +} + +int postcopy_ram_incoming_setup(MigrationIncomingState *mis) +{ + assert(0); + return -1; +} + +int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, + RAMBlock *rb) +{ + assert(0); + return -1; +} + +int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, + RAMBlock *rb) +{ + assert(0); + return -1; +} + +int postcopy_wake_shared(struct PostCopyFD *pcfd, + uint64_t client_addr, + RAMBlock *rb) +{ + assert(0); + return -1; +} +#endif + +/* ------------------------------------------------------------------------- */ + +void postcopy_fault_thread_notify(MigrationIncomingState *mis) +{ + uint64_t tmp64 = 1; + + /* + * Wakeup the fault_thread. It's an eventfd that should currently + * be at 0, we're going to increment it to 1 + */ + if (write(mis->userfault_event_fd, &tmp64, 8) != 8) { + /* Not much we can do here, but may as well report it */ + error_report("%s: incrementing failed: %s", __func__, + strerror(errno)); + } +} + +/** + * postcopy_discard_send_init: Called at the start of each RAMBlock before + * asking to discard individual ranges. + * + * @ms: The current migration state. + * @offset: the bitmap offset of the named RAMBlock in the migration bitmap. + * @name: RAMBlock that discards will operate on. + */ +static PostcopyDiscardState pds = {0}; +void postcopy_discard_send_init(MigrationState *ms, const char *name) +{ + pds.ramblock_name = name; + pds.cur_entry = 0; + pds.nsentwords = 0; + pds.nsentcmds = 0; +} + +/** + * postcopy_discard_send_range: Called by the bitmap code for each chunk to + * discard. May send a discard message, may just leave it queued to + * be sent later. + * + * @ms: Current migration state. + * @start,@length: a range of pages in the migration bitmap in the + * RAM block passed to postcopy_discard_send_init() (length=1 is one page) + */ +void postcopy_discard_send_range(MigrationState *ms, unsigned long start, + unsigned long length) +{ + size_t tp_size = qemu_target_page_size(); + /* Convert to byte offsets within the RAM block */ + pds.start_list[pds.cur_entry] = start * tp_size; + pds.length_list[pds.cur_entry] = length * tp_size; + trace_postcopy_discard_send_range(pds.ramblock_name, start, length); + pds.cur_entry++; + pds.nsentwords++; + + if (pds.cur_entry == MAX_DISCARDS_PER_COMMAND) { + /* Full set, ship it! */ + qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, + pds.ramblock_name, + pds.cur_entry, + pds.start_list, + pds.length_list); + pds.nsentcmds++; + pds.cur_entry = 0; + } +} + +/** + * postcopy_discard_send_finish: Called at the end of each RAMBlock by the + * bitmap code. Sends any outstanding discard messages, frees the PDS + * + * @ms: Current migration state. + */ +void postcopy_discard_send_finish(MigrationState *ms) +{ + /* Anything unsent? */ + if (pds.cur_entry) { + qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, + pds.ramblock_name, + pds.cur_entry, + pds.start_list, + pds.length_list); + pds.nsentcmds++; + } + + trace_postcopy_discard_send_finish(pds.ramblock_name, pds.nsentwords, + pds.nsentcmds); +} + +/* + * Current state of incoming postcopy; note this is not part of + * MigrationIncomingState since it's state is used during cleanup + * at the end as MIS is being freed. + */ +static PostcopyState incoming_postcopy_state; + +PostcopyState postcopy_state_get(void) +{ + return qatomic_mb_read(&incoming_postcopy_state); +} + +/* Set the state and return the old state */ +PostcopyState postcopy_state_set(PostcopyState new_state) +{ + return qatomic_xchg(&incoming_postcopy_state, new_state); +} + +/* Register a handler for external shared memory postcopy + * called on the destination. + */ +void postcopy_register_shared_ufd(struct PostCopyFD *pcfd) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + + mis->postcopy_remote_fds = g_array_append_val(mis->postcopy_remote_fds, + *pcfd); +} + +/* Unregister a handler for external shared memory postcopy + */ +void postcopy_unregister_shared_ufd(struct PostCopyFD *pcfd) +{ + guint i; + MigrationIncomingState *mis = migration_incoming_get_current(); + GArray *pcrfds = mis->postcopy_remote_fds; + + if (!pcrfds) { + /* migration has already finished and freed the array */ + return; + } + for (i = 0; i < pcrfds->len; i++) { + struct PostCopyFD *cur = &g_array_index(pcrfds, struct PostCopyFD, i); + if (cur->fd == pcfd->fd) { + mis->postcopy_remote_fds = g_array_remove_index(pcrfds, i); + return; + } + } +} diff --git a/migration/postcopy-ram.h b/migration/postcopy-ram.h new file mode 100644 index 000000000..6d2b3cf12 --- /dev/null +++ b/migration/postcopy-ram.h @@ -0,0 +1,182 @@ +/* + * Postcopy migration for RAM + * + * Copyright 2013 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Dave Gilbert <dgilbert@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ +#ifndef QEMU_POSTCOPY_RAM_H +#define QEMU_POSTCOPY_RAM_H + +/* Return true if the host supports everything we need to do postcopy-ram */ +bool postcopy_ram_supported_by_host(MigrationIncomingState *mis); + +/* + * Make all of RAM sensitive to accesses to areas that haven't yet been written + * and wire up anything necessary to deal with it. + */ +int postcopy_ram_incoming_setup(MigrationIncomingState *mis); + +/* + * Initialise postcopy-ram, setting the RAM to a state where we can go into + * postcopy later; must be called prior to any precopy. + * called from ram.c's similarly named ram_postcopy_incoming_init + */ +int postcopy_ram_incoming_init(MigrationIncomingState *mis); + +/* + * At the end of a migration where postcopy_ram_incoming_init was called. + */ +int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis); + +/* + * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard + * however leaving it until after precopy means that most of the precopy + * data is still THPd + */ +int postcopy_ram_prepare_discard(MigrationIncomingState *mis); + +/* + * Called at the start of each RAMBlock by the bitmap code. + */ +void postcopy_discard_send_init(MigrationState *ms, const char *name); + +/* + * Called by the bitmap code for each chunk to discard. + * May send a discard message, may just leave it queued to + * be sent later. + * @start,@length: a range of pages in the migration bitmap in the + * RAM block passed to postcopy_discard_send_init() (length=1 is one page) + */ +void postcopy_discard_send_range(MigrationState *ms, unsigned long start, + unsigned long length); + +/* + * Called at the end of each RAMBlock by the bitmap code. + * Sends any outstanding discard messages. + */ +void postcopy_discard_send_finish(MigrationState *ms); + +/* + * Place a page (from) at (host) efficiently + * There are restrictions on how 'from' must be mapped, in general best + * to use other postcopy_ routines to allocate. + * returns 0 on success + */ +int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from, + RAMBlock *rb); + +/* + * Place a zero page at (host) atomically + * returns 0 on success + */ +int postcopy_place_page_zero(MigrationIncomingState *mis, void *host, + RAMBlock *rb); + +/* The current postcopy state is read/set by postcopy_state_get/set + * which update it atomically. + * The state is updated as postcopy messages are received, and + * in general only one thread should be writing to the state at any one + * time, initially the main thread and then the listen thread; + * Corner cases are where either thread finishes early and/or errors. + * The state is checked as messages are received to ensure that + * the source is sending us messages in the correct order. + * The state is also used by the RAM reception code to know if it + * has to place pages atomically, and the cleanup code at the end of + * the main thread to know if it has to delay cleanup until the end + * of postcopy. + */ +typedef enum { + POSTCOPY_INCOMING_NONE = 0, /* Initial state - no postcopy */ + POSTCOPY_INCOMING_ADVISE, + POSTCOPY_INCOMING_DISCARD, + POSTCOPY_INCOMING_LISTENING, + POSTCOPY_INCOMING_RUNNING, + POSTCOPY_INCOMING_END +} PostcopyState; + +PostcopyState postcopy_state_get(void); +/* Set the state and return the old state */ +PostcopyState postcopy_state_set(PostcopyState new_state); + +void postcopy_fault_thread_notify(MigrationIncomingState *mis); + +/* + * To be called once at the start before any device initialisation + */ +void postcopy_infrastructure_init(void); + +/* Add a notifier to a list to be called when checking whether the devices + * can support postcopy. + * It's data is a *PostcopyNotifyData + * It should return 0 if OK, or a negative value on failure. + * On failure it must set the data->errp to an error. + * + */ +enum PostcopyNotifyReason { + POSTCOPY_NOTIFY_PROBE = 0, + POSTCOPY_NOTIFY_INBOUND_ADVISE, + POSTCOPY_NOTIFY_INBOUND_LISTEN, + POSTCOPY_NOTIFY_INBOUND_END, +}; + +struct PostcopyNotifyData { + enum PostcopyNotifyReason reason; + Error **errp; +}; + +void postcopy_add_notifier(NotifierWithReturn *nn); +void postcopy_remove_notifier(NotifierWithReturn *n); +/* Call the notifier list set by postcopy_add_start_notifier */ +int postcopy_notify(enum PostcopyNotifyReason reason, Error **errp); + +struct PostCopyFD; + +/* ufd is a pointer to the struct uffd_msg *TODO: more Portable! */ +typedef int (*pcfdhandler)(struct PostCopyFD *pcfd, void *ufd); +/* Notification to wake, either on place or on reception of + * a fault on something that's already arrived (race) + */ +typedef int (*pcfdwake)(struct PostCopyFD *pcfd, RAMBlock *rb, uint64_t offset); + +struct PostCopyFD { + int fd; + /* Data to pass to handler */ + void *data; + /* Handler to be called whenever we get a poll event */ + pcfdhandler handler; + /* Notification to wake shared client */ + pcfdwake waker; + /* A string to use in error messages */ + const char *idstr; +}; + +/* Register a userfaultfd owned by an external process for + * shared memory. + */ +void postcopy_register_shared_ufd(struct PostCopyFD *pcfd); +void postcopy_unregister_shared_ufd(struct PostCopyFD *pcfd); +/* Call each of the shared 'waker's registered telling them of + * availability of a block. + */ +int postcopy_notify_shared_wake(RAMBlock *rb, uint64_t offset); +/* postcopy_wake_shared: Notify a client ufd that a page is available + * + * Returns 0 on success + * + * @pcfd: Structure with fd, handler and name as above + * @client_addr: Address in the client program, not QEMU + * @rb: The RAMBlock the page is in + */ +int postcopy_wake_shared(struct PostCopyFD *pcfd, uint64_t client_addr, + RAMBlock *rb); +/* Callback from shared fault handlers to ask for a page */ +int postcopy_request_shared_page(struct PostCopyFD *pcfd, RAMBlock *rb, + uint64_t client_addr, uint64_t offset); + +#endif diff --git a/migration/qemu-file-channel.c b/migration/qemu-file-channel.c new file mode 100644 index 000000000..bb5a5752d --- /dev/null +++ b/migration/qemu-file-channel.c @@ -0,0 +1,194 @@ +/* + * QEMUFile backend for QIOChannel objects + * + * Copyright (c) 2015-2016 Red Hat, Inc + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "qemu/osdep.h" +#include "qemu-file-channel.h" +#include "qemu-file.h" +#include "io/channel-socket.h" +#include "io/channel-tls.h" +#include "qemu/iov.h" +#include "qemu/yank.h" +#include "yank_functions.h" + + +static ssize_t channel_writev_buffer(void *opaque, + struct iovec *iov, + int iovcnt, + int64_t pos, + Error **errp) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + ssize_t done = 0; + struct iovec *local_iov = g_new(struct iovec, iovcnt); + struct iovec *local_iov_head = local_iov; + unsigned int nlocal_iov = iovcnt; + + nlocal_iov = iov_copy(local_iov, nlocal_iov, + iov, iovcnt, + 0, iov_size(iov, iovcnt)); + + while (nlocal_iov > 0) { + ssize_t len; + len = qio_channel_writev(ioc, local_iov, nlocal_iov, errp); + if (len == QIO_CHANNEL_ERR_BLOCK) { + if (qemu_in_coroutine()) { + qio_channel_yield(ioc, G_IO_OUT); + } else { + qio_channel_wait(ioc, G_IO_OUT); + } + continue; + } + if (len < 0) { + done = -EIO; + goto cleanup; + } + + iov_discard_front(&local_iov, &nlocal_iov, len); + done += len; + } + + cleanup: + g_free(local_iov_head); + return done; +} + + +static ssize_t channel_get_buffer(void *opaque, + uint8_t *buf, + int64_t pos, + size_t size, + Error **errp) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + ssize_t ret; + + do { + ret = qio_channel_read(ioc, (char *)buf, size, errp); + if (ret < 0) { + if (ret == QIO_CHANNEL_ERR_BLOCK) { + if (qemu_in_coroutine()) { + qio_channel_yield(ioc, G_IO_IN); + } else { + qio_channel_wait(ioc, G_IO_IN); + } + } else { + return -EIO; + } + } + } while (ret == QIO_CHANNEL_ERR_BLOCK); + + return ret; +} + + +static int channel_close(void *opaque, Error **errp) +{ + int ret; + QIOChannel *ioc = QIO_CHANNEL(opaque); + ret = qio_channel_close(ioc, errp); + object_unref(OBJECT(ioc)); + return ret; +} + + +static int channel_shutdown(void *opaque, + bool rd, + bool wr, + Error **errp) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + + if (qio_channel_has_feature(ioc, + QIO_CHANNEL_FEATURE_SHUTDOWN)) { + QIOChannelShutdown mode; + if (rd && wr) { + mode = QIO_CHANNEL_SHUTDOWN_BOTH; + } else if (rd) { + mode = QIO_CHANNEL_SHUTDOWN_READ; + } else { + mode = QIO_CHANNEL_SHUTDOWN_WRITE; + } + if (qio_channel_shutdown(ioc, mode, errp) < 0) { + return -EIO; + } + } + return 0; +} + + +static int channel_set_blocking(void *opaque, + bool enabled, + Error **errp) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + + if (qio_channel_set_blocking(ioc, enabled, errp) < 0) { + return -1; + } + return 0; +} + +static QEMUFile *channel_get_input_return_path(void *opaque) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + + return qemu_fopen_channel_output(ioc); +} + +static QEMUFile *channel_get_output_return_path(void *opaque) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + + return qemu_fopen_channel_input(ioc); +} + +static const QEMUFileOps channel_input_ops = { + .get_buffer = channel_get_buffer, + .close = channel_close, + .shut_down = channel_shutdown, + .set_blocking = channel_set_blocking, + .get_return_path = channel_get_input_return_path, +}; + + +static const QEMUFileOps channel_output_ops = { + .writev_buffer = channel_writev_buffer, + .close = channel_close, + .shut_down = channel_shutdown, + .set_blocking = channel_set_blocking, + .get_return_path = channel_get_output_return_path, +}; + + +QEMUFile *qemu_fopen_channel_input(QIOChannel *ioc) +{ + object_ref(OBJECT(ioc)); + return qemu_fopen_ops(ioc, &channel_input_ops, true); +} + +QEMUFile *qemu_fopen_channel_output(QIOChannel *ioc) +{ + object_ref(OBJECT(ioc)); + return qemu_fopen_ops(ioc, &channel_output_ops, true); +} diff --git a/migration/qemu-file-channel.h b/migration/qemu-file-channel.h new file mode 100644 index 000000000..0028a09eb --- /dev/null +++ b/migration/qemu-file-channel.h @@ -0,0 +1,32 @@ +/* + * QEMUFile backend for QIOChannel objects + * + * Copyright (c) 2015-2016 Red Hat, Inc + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef QEMU_FILE_CHANNEL_H +#define QEMU_FILE_CHANNEL_H + +#include "io/channel.h" + +QEMUFile *qemu_fopen_channel_input(QIOChannel *ioc); +QEMUFile *qemu_fopen_channel_output(QIOChannel *ioc); +#endif diff --git a/migration/qemu-file.c b/migration/qemu-file.c new file mode 100644 index 000000000..6338d8e2f --- /dev/null +++ b/migration/qemu-file.c @@ -0,0 +1,868 @@ +/* + * QEMU System Emulator + * + * Copyright (c) 2003-2008 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "qemu/osdep.h" +#include <zlib.h> +#include "qemu/error-report.h" +#include "qemu/iov.h" +#include "migration.h" +#include "qemu-file.h" +#include "trace.h" +#include "qapi/error.h" + +#define IO_BUF_SIZE 32768 +#define MAX_IOV_SIZE MIN_CONST(IOV_MAX, 64) + +struct QEMUFile { + const QEMUFileOps *ops; + const QEMUFileHooks *hooks; + void *opaque; + + int64_t bytes_xfer; + int64_t xfer_limit; + + int64_t pos; /* start of buffer when writing, end of buffer + when reading */ + int buf_index; + int buf_size; /* 0 when writing */ + uint8_t buf[IO_BUF_SIZE]; + + DECLARE_BITMAP(may_free, MAX_IOV_SIZE); + struct iovec iov[MAX_IOV_SIZE]; + unsigned int iovcnt; + + int last_error; + Error *last_error_obj; + /* has the file has been shutdown */ + bool shutdown; + /* Whether opaque points to a QIOChannel */ + bool has_ioc; +}; + +/* + * Stop a file from being read/written - not all backing files can do this + * typically only sockets can. + */ +int qemu_file_shutdown(QEMUFile *f) +{ + int ret; + + f->shutdown = true; + if (!f->ops->shut_down) { + return -ENOSYS; + } + ret = f->ops->shut_down(f->opaque, true, true, NULL); + + if (!f->last_error) { + qemu_file_set_error(f, -EIO); + } + return ret; +} + +/* + * Result: QEMUFile* for a 'return path' for comms in the opposite direction + * NULL if not available + */ +QEMUFile *qemu_file_get_return_path(QEMUFile *f) +{ + if (!f->ops->get_return_path) { + return NULL; + } + return f->ops->get_return_path(f->opaque); +} + +bool qemu_file_mode_is_not_valid(const char *mode) +{ + if (mode == NULL || + (mode[0] != 'r' && mode[0] != 'w') || + mode[1] != 'b' || mode[2] != 0) { + fprintf(stderr, "qemu_fopen: Argument validity check failed\n"); + return true; + } + + return false; +} + +QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops, bool has_ioc) +{ + QEMUFile *f; + + f = g_new0(QEMUFile, 1); + + f->opaque = opaque; + f->ops = ops; + f->has_ioc = has_ioc; + return f; +} + + +void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks) +{ + f->hooks = hooks; +} + +/* + * Get last error for stream f with optional Error* + * + * Return negative error value if there has been an error on previous + * operations, return 0 if no error happened. + * Optional, it returns Error* in errp, but it may be NULL even if return value + * is not 0. + * + */ +int qemu_file_get_error_obj(QEMUFile *f, Error **errp) +{ + if (errp) { + *errp = f->last_error_obj ? error_copy(f->last_error_obj) : NULL; + } + return f->last_error; +} + +/* + * Set the last error for stream f with optional Error* + */ +void qemu_file_set_error_obj(QEMUFile *f, int ret, Error *err) +{ + if (f->last_error == 0 && ret) { + f->last_error = ret; + error_propagate(&f->last_error_obj, err); + } else if (err) { + error_report_err(err); + } +} + +/* + * Get last error for stream f + * + * Return negative error value if there has been an error on previous + * operations, return 0 if no error happened. + * + */ +int qemu_file_get_error(QEMUFile *f) +{ + return qemu_file_get_error_obj(f, NULL); +} + +/* + * Set the last error for stream f + */ +void qemu_file_set_error(QEMUFile *f, int ret) +{ + qemu_file_set_error_obj(f, ret, NULL); +} + +bool qemu_file_is_writable(QEMUFile *f) +{ + return f->ops->writev_buffer; +} + +static void qemu_iovec_release_ram(QEMUFile *f) +{ + struct iovec iov; + unsigned long idx; + + /* Find and release all the contiguous memory ranges marked as may_free. */ + idx = find_next_bit(f->may_free, f->iovcnt, 0); + if (idx >= f->iovcnt) { + return; + } + iov = f->iov[idx]; + + /* The madvise() in the loop is called for iov within a continuous range and + * then reinitialize the iov. And in the end, madvise() is called for the + * last iov. + */ + while ((idx = find_next_bit(f->may_free, f->iovcnt, idx + 1)) < f->iovcnt) { + /* check for adjacent buffer and coalesce them */ + if (iov.iov_base + iov.iov_len == f->iov[idx].iov_base) { + iov.iov_len += f->iov[idx].iov_len; + continue; + } + if (qemu_madvise(iov.iov_base, iov.iov_len, QEMU_MADV_DONTNEED) < 0) { + error_report("migrate: madvise DONTNEED failed %p %zd: %s", + iov.iov_base, iov.iov_len, strerror(errno)); + } + iov = f->iov[idx]; + } + if (qemu_madvise(iov.iov_base, iov.iov_len, QEMU_MADV_DONTNEED) < 0) { + error_report("migrate: madvise DONTNEED failed %p %zd: %s", + iov.iov_base, iov.iov_len, strerror(errno)); + } + memset(f->may_free, 0, sizeof(f->may_free)); +} + +/** + * Flushes QEMUFile buffer + * + * This will flush all pending data. If data was only partially flushed, it + * will set an error state. + */ +void qemu_fflush(QEMUFile *f) +{ + ssize_t ret = 0; + ssize_t expect = 0; + Error *local_error = NULL; + + if (!qemu_file_is_writable(f)) { + return; + } + + if (f->shutdown) { + return; + } + if (f->iovcnt > 0) { + expect = iov_size(f->iov, f->iovcnt); + ret = f->ops->writev_buffer(f->opaque, f->iov, f->iovcnt, f->pos, + &local_error); + + qemu_iovec_release_ram(f); + } + + if (ret >= 0) { + f->pos += ret; + } + /* We expect the QEMUFile write impl to send the full + * data set we requested, so sanity check that. + */ + if (ret != expect) { + qemu_file_set_error_obj(f, ret < 0 ? ret : -EIO, local_error); + } + f->buf_index = 0; + f->iovcnt = 0; +} + +void ram_control_before_iterate(QEMUFile *f, uint64_t flags) +{ + int ret = 0; + + if (f->hooks && f->hooks->before_ram_iterate) { + ret = f->hooks->before_ram_iterate(f, f->opaque, flags, NULL); + if (ret < 0) { + qemu_file_set_error(f, ret); + } + } +} + +void ram_control_after_iterate(QEMUFile *f, uint64_t flags) +{ + int ret = 0; + + if (f->hooks && f->hooks->after_ram_iterate) { + ret = f->hooks->after_ram_iterate(f, f->opaque, flags, NULL); + if (ret < 0) { + qemu_file_set_error(f, ret); + } + } +} + +void ram_control_load_hook(QEMUFile *f, uint64_t flags, void *data) +{ + int ret = -EINVAL; + + if (f->hooks && f->hooks->hook_ram_load) { + ret = f->hooks->hook_ram_load(f, f->opaque, flags, data); + if (ret < 0) { + qemu_file_set_error(f, ret); + } + } else { + /* + * Hook is a hook specifically requested by the source sending a flag + * that expects there to be a hook on the destination. + */ + if (flags == RAM_CONTROL_HOOK) { + qemu_file_set_error(f, ret); + } + } +} + +size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset, + ram_addr_t offset, size_t size, + uint64_t *bytes_sent) +{ + if (f->hooks && f->hooks->save_page) { + int ret = f->hooks->save_page(f, f->opaque, block_offset, + offset, size, bytes_sent); + if (ret != RAM_SAVE_CONTROL_NOT_SUPP) { + f->bytes_xfer += size; + } + + if (ret != RAM_SAVE_CONTROL_DELAYED && + ret != RAM_SAVE_CONTROL_NOT_SUPP) { + if (bytes_sent && *bytes_sent > 0) { + qemu_update_position(f, *bytes_sent); + } else if (ret < 0) { + qemu_file_set_error(f, ret); + } + } + + return ret; + } + + return RAM_SAVE_CONTROL_NOT_SUPP; +} + +/* + * Attempt to fill the buffer from the underlying file + * Returns the number of bytes read, or negative value for an error. + * + * Note that it can return a partially full buffer even in a not error/not EOF + * case if the underlying file descriptor gives a short read, and that can + * happen even on a blocking fd. + */ +static ssize_t qemu_fill_buffer(QEMUFile *f) +{ + int len; + int pending; + Error *local_error = NULL; + + assert(!qemu_file_is_writable(f)); + + pending = f->buf_size - f->buf_index; + if (pending > 0) { + memmove(f->buf, f->buf + f->buf_index, pending); + } + f->buf_index = 0; + f->buf_size = pending; + + if (f->shutdown) { + return 0; + } + + len = f->ops->get_buffer(f->opaque, f->buf + pending, f->pos, + IO_BUF_SIZE - pending, &local_error); + if (len > 0) { + f->buf_size += len; + f->pos += len; + } else if (len == 0) { + qemu_file_set_error_obj(f, -EIO, local_error); + } else if (len != -EAGAIN) { + qemu_file_set_error_obj(f, len, local_error); + } else { + error_free(local_error); + } + + return len; +} + +void qemu_update_position(QEMUFile *f, size_t size) +{ + f->pos += size; +} + +/** Closes the file + * + * Returns negative error value if any error happened on previous operations or + * while closing the file. Returns 0 or positive number on success. + * + * The meaning of return value on success depends on the specific backend + * being used. + */ +int qemu_fclose(QEMUFile *f) +{ + int ret; + qemu_fflush(f); + ret = qemu_file_get_error(f); + + if (f->ops->close) { + int ret2 = f->ops->close(f->opaque, NULL); + if (ret >= 0) { + ret = ret2; + } + } + /* If any error was spotted before closing, we should report it + * instead of the close() return value. + */ + if (f->last_error) { + ret = f->last_error; + } + error_free(f->last_error_obj); + g_free(f); + trace_qemu_file_fclose(); + return ret; +} + +/* + * Add buf to iovec. Do flush if iovec is full. + * + * Return values: + * 1 iovec is full and flushed + * 0 iovec is not flushed + * + */ +static int add_to_iovec(QEMUFile *f, const uint8_t *buf, size_t size, + bool may_free) +{ + /* check for adjacent buffer and coalesce them */ + if (f->iovcnt > 0 && buf == f->iov[f->iovcnt - 1].iov_base + + f->iov[f->iovcnt - 1].iov_len && + may_free == test_bit(f->iovcnt - 1, f->may_free)) + { + f->iov[f->iovcnt - 1].iov_len += size; + } else { + if (f->iovcnt >= MAX_IOV_SIZE) { + /* Should only happen if a previous fflush failed */ + assert(f->shutdown || !qemu_file_is_writable(f)); + return 1; + } + if (may_free) { + set_bit(f->iovcnt, f->may_free); + } + f->iov[f->iovcnt].iov_base = (uint8_t *)buf; + f->iov[f->iovcnt++].iov_len = size; + } + + if (f->iovcnt >= MAX_IOV_SIZE) { + qemu_fflush(f); + return 1; + } + + return 0; +} + +static void add_buf_to_iovec(QEMUFile *f, size_t len) +{ + if (!add_to_iovec(f, f->buf + f->buf_index, len, false)) { + f->buf_index += len; + if (f->buf_index == IO_BUF_SIZE) { + qemu_fflush(f); + } + } +} + +void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size, + bool may_free) +{ + if (f->last_error) { + return; + } + + f->bytes_xfer += size; + add_to_iovec(f, buf, size, may_free); +} + +void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size) +{ + size_t l; + + if (f->last_error) { + return; + } + + while (size > 0) { + l = IO_BUF_SIZE - f->buf_index; + if (l > size) { + l = size; + } + memcpy(f->buf + f->buf_index, buf, l); + f->bytes_xfer += l; + add_buf_to_iovec(f, l); + if (qemu_file_get_error(f)) { + break; + } + buf += l; + size -= l; + } +} + +void qemu_put_byte(QEMUFile *f, int v) +{ + if (f->last_error) { + return; + } + + f->buf[f->buf_index] = v; + f->bytes_xfer++; + add_buf_to_iovec(f, 1); +} + +void qemu_file_skip(QEMUFile *f, int size) +{ + if (f->buf_index + size <= f->buf_size) { + f->buf_index += size; + } +} + +/* + * Read 'size' bytes from file (at 'offset') without moving the + * pointer and set 'buf' to point to that data. + * + * It will return size bytes unless there was an error, in which case it will + * return as many as it managed to read (assuming blocking fd's which + * all current QEMUFile are) + */ +size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset) +{ + ssize_t pending; + size_t index; + + assert(!qemu_file_is_writable(f)); + assert(offset < IO_BUF_SIZE); + assert(size <= IO_BUF_SIZE - offset); + + /* The 1st byte to read from */ + index = f->buf_index + offset; + /* The number of available bytes starting at index */ + pending = f->buf_size - index; + + /* + * qemu_fill_buffer might return just a few bytes, even when there isn't + * an error, so loop collecting them until we get enough. + */ + while (pending < size) { + int received = qemu_fill_buffer(f); + + if (received <= 0) { + break; + } + + index = f->buf_index + offset; + pending = f->buf_size - index; + } + + if (pending <= 0) { + return 0; + } + if (size > pending) { + size = pending; + } + + *buf = f->buf + index; + return size; +} + +/* + * Read 'size' bytes of data from the file into buf. + * 'size' can be larger than the internal buffer. + * + * It will return size bytes unless there was an error, in which case it will + * return as many as it managed to read (assuming blocking fd's which + * all current QEMUFile are) + */ +size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size) +{ + size_t pending = size; + size_t done = 0; + + while (pending > 0) { + size_t res; + uint8_t *src; + + res = qemu_peek_buffer(f, &src, MIN(pending, IO_BUF_SIZE), 0); + if (res == 0) { + return done; + } + memcpy(buf, src, res); + qemu_file_skip(f, res); + buf += res; + pending -= res; + done += res; + } + return done; +} + +/* + * Read 'size' bytes of data from the file. + * 'size' can be larger than the internal buffer. + * + * The data: + * may be held on an internal buffer (in which case *buf is updated + * to point to it) that is valid until the next qemu_file operation. + * OR + * will be copied to the *buf that was passed in. + * + * The code tries to avoid the copy if possible. + * + * It will return size bytes unless there was an error, in which case it will + * return as many as it managed to read (assuming blocking fd's which + * all current QEMUFile are) + * + * Note: Since **buf may get changed, the caller should take care to + * keep a pointer to the original buffer if it needs to deallocate it. + */ +size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size) +{ + if (size < IO_BUF_SIZE) { + size_t res; + uint8_t *src = NULL; + + res = qemu_peek_buffer(f, &src, size, 0); + + if (res == size) { + qemu_file_skip(f, res); + *buf = src; + return res; + } + } + + return qemu_get_buffer(f, *buf, size); +} + +/* + * Peeks a single byte from the buffer; this isn't guaranteed to work if + * offset leaves a gap after the previous read/peeked data. + */ +int qemu_peek_byte(QEMUFile *f, int offset) +{ + int index = f->buf_index + offset; + + assert(!qemu_file_is_writable(f)); + assert(offset < IO_BUF_SIZE); + + if (index >= f->buf_size) { + qemu_fill_buffer(f); + index = f->buf_index + offset; + if (index >= f->buf_size) { + return 0; + } + } + return f->buf[index]; +} + +int qemu_get_byte(QEMUFile *f) +{ + int result; + + result = qemu_peek_byte(f, 0); + qemu_file_skip(f, 1); + return result; +} + +int64_t qemu_ftell_fast(QEMUFile *f) +{ + int64_t ret = f->pos; + int i; + + for (i = 0; i < f->iovcnt; i++) { + ret += f->iov[i].iov_len; + } + + return ret; +} + +int64_t qemu_ftell(QEMUFile *f) +{ + qemu_fflush(f); + return f->pos; +} + +int qemu_file_rate_limit(QEMUFile *f) +{ + if (f->shutdown) { + return 1; + } + if (qemu_file_get_error(f)) { + return 1; + } + if (f->xfer_limit > 0 && f->bytes_xfer > f->xfer_limit) { + return 1; + } + return 0; +} + +int64_t qemu_file_get_rate_limit(QEMUFile *f) +{ + return f->xfer_limit; +} + +void qemu_file_set_rate_limit(QEMUFile *f, int64_t limit) +{ + f->xfer_limit = limit; +} + +void qemu_file_reset_rate_limit(QEMUFile *f) +{ + f->bytes_xfer = 0; +} + +void qemu_file_update_transfer(QEMUFile *f, int64_t len) +{ + f->bytes_xfer += len; +} + +void qemu_put_be16(QEMUFile *f, unsigned int v) +{ + qemu_put_byte(f, v >> 8); + qemu_put_byte(f, v); +} + +void qemu_put_be32(QEMUFile *f, unsigned int v) +{ + qemu_put_byte(f, v >> 24); + qemu_put_byte(f, v >> 16); + qemu_put_byte(f, v >> 8); + qemu_put_byte(f, v); +} + +void qemu_put_be64(QEMUFile *f, uint64_t v) +{ + qemu_put_be32(f, v >> 32); + qemu_put_be32(f, v); +} + +unsigned int qemu_get_be16(QEMUFile *f) +{ + unsigned int v; + v = qemu_get_byte(f) << 8; + v |= qemu_get_byte(f); + return v; +} + +unsigned int qemu_get_be32(QEMUFile *f) +{ + unsigned int v; + v = (unsigned int)qemu_get_byte(f) << 24; + v |= qemu_get_byte(f) << 16; + v |= qemu_get_byte(f) << 8; + v |= qemu_get_byte(f); + return v; +} + +uint64_t qemu_get_be64(QEMUFile *f) +{ + uint64_t v; + v = (uint64_t)qemu_get_be32(f) << 32; + v |= qemu_get_be32(f); + return v; +} + +/* return the size after compression, or negative value on error */ +static int qemu_compress_data(z_stream *stream, uint8_t *dest, size_t dest_len, + const uint8_t *source, size_t source_len) +{ + int err; + + err = deflateReset(stream); + if (err != Z_OK) { + return -1; + } + + stream->avail_in = source_len; + stream->next_in = (uint8_t *)source; + stream->avail_out = dest_len; + stream->next_out = dest; + + err = deflate(stream, Z_FINISH); + if (err != Z_STREAM_END) { + return -1; + } + + return stream->next_out - dest; +} + +/* Compress size bytes of data start at p and store the compressed + * data to the buffer of f. + * + * Since the file is dummy file with empty_ops, return -1 if f has no space to + * save the compressed data. + */ +ssize_t qemu_put_compression_data(QEMUFile *f, z_stream *stream, + const uint8_t *p, size_t size) +{ + ssize_t blen = IO_BUF_SIZE - f->buf_index - sizeof(int32_t); + + if (blen < compressBound(size)) { + return -1; + } + + blen = qemu_compress_data(stream, f->buf + f->buf_index + sizeof(int32_t), + blen, p, size); + if (blen < 0) { + return -1; + } + + qemu_put_be32(f, blen); + add_buf_to_iovec(f, blen); + return blen + sizeof(int32_t); +} + +/* Put the data in the buffer of f_src to the buffer of f_des, and + * then reset the buf_index of f_src to 0. + */ + +int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src) +{ + int len = 0; + + if (f_src->buf_index > 0) { + len = f_src->buf_index; + qemu_put_buffer(f_des, f_src->buf, f_src->buf_index); + f_src->buf_index = 0; + f_src->iovcnt = 0; + } + return len; +} + +/* + * Get a string whose length is determined by a single preceding byte + * A preallocated 256 byte buffer must be passed in. + * Returns: len on success and a 0 terminated string in the buffer + * else 0 + * (Note a 0 length string will return 0 either way) + */ +size_t qemu_get_counted_string(QEMUFile *f, char buf[256]) +{ + size_t len = qemu_get_byte(f); + size_t res = qemu_get_buffer(f, (uint8_t *)buf, len); + + buf[res] = 0; + + return res == len ? res : 0; +} + +/* + * Put a string with one preceding byte containing its length. The length of + * the string should be less than 256. + */ +void qemu_put_counted_string(QEMUFile *f, const char *str) +{ + size_t len = strlen(str); + + assert(len < 256); + qemu_put_byte(f, len); + qemu_put_buffer(f, (const uint8_t *)str, len); +} + +/* + * Set the blocking state of the QEMUFile. + * Note: On some transports the OS only keeps a single blocking state for + * both directions, and thus changing the blocking on the main + * QEMUFile can also affect the return path. + */ +void qemu_file_set_blocking(QEMUFile *f, bool block) +{ + if (f->ops->set_blocking) { + f->ops->set_blocking(f->opaque, block, NULL); + } +} + +/* + * Return the ioc object if it's a migration channel. Note: it can return NULL + * for callers passing in a non-migration qemufile. E.g. see qemu_fopen_bdrv() + * and its usage in e.g. load_snapshot(). So we need to check against NULL + * before using it. If without the check, migration_incoming_state_destroy() + * could fail for load_snapshot(). + */ +QIOChannel *qemu_file_get_ioc(QEMUFile *file) +{ + return file->has_ioc ? QIO_CHANNEL(file->opaque) : NULL; +} diff --git a/migration/qemu-file.h b/migration/qemu-file.h new file mode 100644 index 000000000..3f36d4dc8 --- /dev/null +++ b/migration/qemu-file.h @@ -0,0 +1,185 @@ +/* + * QEMU System Emulator + * + * Copyright (c) 2003-2008 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef MIGRATION_QEMU_FILE_H +#define MIGRATION_QEMU_FILE_H + +#include <zlib.h> +#include "exec/cpu-common.h" +#include "io/channel.h" + +/* Read a chunk of data from a file at the given position. The pos argument + * can be ignored if the file is only be used for streaming. The number of + * bytes actually read should be returned. + */ +typedef ssize_t (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf, + int64_t pos, size_t size, + Error **errp); + +/* Close a file + * + * Return negative error number on error, 0 or positive value on success. + * + * The meaning of return value on success depends on the specific back-end being + * used. + */ +typedef int (QEMUFileCloseFunc)(void *opaque, Error **errp); + +/* Called to return the OS file descriptor associated to the QEMUFile. + */ +typedef int (QEMUFileGetFD)(void *opaque); + +/* Called to change the blocking mode of the file + */ +typedef int (QEMUFileSetBlocking)(void *opaque, bool enabled, Error **errp); + +/* + * This function writes an iovec to file. The handler must write all + * of the data or return a negative errno value. + */ +typedef ssize_t (QEMUFileWritevBufferFunc)(void *opaque, struct iovec *iov, + int iovcnt, int64_t pos, + Error **errp); + +/* + * This function provides hooks around different + * stages of RAM migration. + * 'opaque' is the backend specific data in QEMUFile + * 'data' is call specific data associated with the 'flags' value + */ +typedef int (QEMURamHookFunc)(QEMUFile *f, void *opaque, uint64_t flags, + void *data); + +/* + * Constants used by ram_control_* hooks + */ +#define RAM_CONTROL_SETUP 0 +#define RAM_CONTROL_ROUND 1 +#define RAM_CONTROL_HOOK 2 +#define RAM_CONTROL_FINISH 3 +#define RAM_CONTROL_BLOCK_REG 4 + +/* + * This function allows override of where the RAM page + * is saved (such as RDMA, for example.) + */ +typedef size_t (QEMURamSaveFunc)(QEMUFile *f, void *opaque, + ram_addr_t block_offset, + ram_addr_t offset, + size_t size, + uint64_t *bytes_sent); + +/* + * Return a QEMUFile for comms in the opposite direction + */ +typedef QEMUFile *(QEMURetPathFunc)(void *opaque); + +/* + * Stop any read or write (depending on flags) on the underlying + * transport on the QEMUFile. + * Existing blocking reads/writes must be woken + * Returns 0 on success, -err on error + */ +typedef int (QEMUFileShutdownFunc)(void *opaque, bool rd, bool wr, + Error **errp); + +typedef struct QEMUFileOps { + QEMUFileGetBufferFunc *get_buffer; + QEMUFileCloseFunc *close; + QEMUFileSetBlocking *set_blocking; + QEMUFileWritevBufferFunc *writev_buffer; + QEMURetPathFunc *get_return_path; + QEMUFileShutdownFunc *shut_down; +} QEMUFileOps; + +typedef struct QEMUFileHooks { + QEMURamHookFunc *before_ram_iterate; + QEMURamHookFunc *after_ram_iterate; + QEMURamHookFunc *hook_ram_load; + QEMURamSaveFunc *save_page; +} QEMUFileHooks; + +QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops, bool has_ioc); +void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks); +int qemu_get_fd(QEMUFile *f); +int qemu_fclose(QEMUFile *f); +int64_t qemu_ftell(QEMUFile *f); +int64_t qemu_ftell_fast(QEMUFile *f); +/* + * put_buffer without copying the buffer. + * The buffer should be available till it is sent asynchronously. + */ +void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size, + bool may_free); +bool qemu_file_mode_is_not_valid(const char *mode); +bool qemu_file_is_writable(QEMUFile *f); + +#include "migration/qemu-file-types.h" + +size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset); +size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size); +ssize_t qemu_put_compression_data(QEMUFile *f, z_stream *stream, + const uint8_t *p, size_t size); +int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src); + +/* + * Note that you can only peek continuous bytes from where the current pointer + * is; you aren't guaranteed to be able to peak to +n bytes unless you've + * previously peeked +n-1. + */ +int qemu_peek_byte(QEMUFile *f, int offset); +void qemu_file_skip(QEMUFile *f, int size); +void qemu_update_position(QEMUFile *f, size_t size); +void qemu_file_reset_rate_limit(QEMUFile *f); +void qemu_file_update_transfer(QEMUFile *f, int64_t len); +void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate); +int64_t qemu_file_get_rate_limit(QEMUFile *f); +int qemu_file_get_error_obj(QEMUFile *f, Error **errp); +void qemu_file_set_error_obj(QEMUFile *f, int ret, Error *err); +void qemu_file_set_error(QEMUFile *f, int ret); +int qemu_file_shutdown(QEMUFile *f); +QEMUFile *qemu_file_get_return_path(QEMUFile *f); +void qemu_fflush(QEMUFile *f); +void qemu_file_set_blocking(QEMUFile *f, bool block); + +void ram_control_before_iterate(QEMUFile *f, uint64_t flags); +void ram_control_after_iterate(QEMUFile *f, uint64_t flags); +void ram_control_load_hook(QEMUFile *f, uint64_t flags, void *data); + +/* Whenever this is found in the data stream, the flags + * will be passed to ram_control_load_hook in the incoming-migration + * side. This lets before_ram_iterate/after_ram_iterate add + * transport-specific sections to the RAM migration data. + */ +#define RAM_SAVE_FLAG_HOOK 0x80 + +#define RAM_SAVE_CONTROL_NOT_SUPP -1000 +#define RAM_SAVE_CONTROL_DELAYED -2000 + +size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset, + ram_addr_t offset, size_t size, + uint64_t *bytes_sent); +QIOChannel *qemu_file_get_ioc(QEMUFile *file); + +#endif diff --git a/migration/ram.c b/migration/ram.c new file mode 100644 index 000000000..863035d23 --- /dev/null +++ b/migration/ram.c @@ -0,0 +1,4427 @@ +/* + * QEMU System Emulator + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2011-2015 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "qemu/osdep.h" +#include "qemu/cutils.h" +#include "qemu/bitops.h" +#include "qemu/bitmap.h" +#include "qemu/main-loop.h" +#include "xbzrle.h" +#include "ram.h" +#include "migration.h" +#include "migration/register.h" +#include "migration/misc.h" +#include "qemu-file.h" +#include "postcopy-ram.h" +#include "page_cache.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "qapi/qapi-types-migration.h" +#include "qapi/qapi-events-migration.h" +#include "qapi/qmp/qerror.h" +#include "trace.h" +#include "exec/ram_addr.h" +#include "exec/target_page.h" +#include "qemu/rcu_queue.h" +#include "migration/colo.h" +#include "block.h" +#include "sysemu/cpu-throttle.h" +#include "savevm.h" +#include "qemu/iov.h" +#include "multifd.h" +#include "sysemu/runstate.h" + +#include "hw/boards.h" /* for machine_dump_guest_core() */ + +#if defined(__linux__) +#include "qemu/userfaultfd.h" +#endif /* defined(__linux__) */ + +/***********************************************************/ +/* ram save/restore */ + +/* RAM_SAVE_FLAG_ZERO used to be named RAM_SAVE_FLAG_COMPRESS, it + * worked for pages that where filled with the same char. We switched + * it to only search for the zero value. And to avoid confusion with + * RAM_SSAVE_FLAG_COMPRESS_PAGE just rename it. + */ + +#define RAM_SAVE_FLAG_FULL 0x01 /* Obsolete, not used anymore */ +#define RAM_SAVE_FLAG_ZERO 0x02 +#define RAM_SAVE_FLAG_MEM_SIZE 0x04 +#define RAM_SAVE_FLAG_PAGE 0x08 +#define RAM_SAVE_FLAG_EOS 0x10 +#define RAM_SAVE_FLAG_CONTINUE 0x20 +#define RAM_SAVE_FLAG_XBZRLE 0x40 +/* 0x80 is reserved in migration.h start with 0x100 next */ +#define RAM_SAVE_FLAG_COMPRESS_PAGE 0x100 + +static inline bool is_zero_range(uint8_t *p, uint64_t size) +{ + return buffer_is_zero(p, size); +} + +XBZRLECacheStats xbzrle_counters; + +/* struct contains XBZRLE cache and a static page + used by the compression */ +static struct { + /* buffer used for XBZRLE encoding */ + uint8_t *encoded_buf; + /* buffer for storing page content */ + uint8_t *current_buf; + /* Cache for XBZRLE, Protected by lock. */ + PageCache *cache; + QemuMutex lock; + /* it will store a page full of zeros */ + uint8_t *zero_target_page; + /* buffer used for XBZRLE decoding */ + uint8_t *decoded_buf; +} XBZRLE; + +static void XBZRLE_cache_lock(void) +{ + if (migrate_use_xbzrle()) { + qemu_mutex_lock(&XBZRLE.lock); + } +} + +static void XBZRLE_cache_unlock(void) +{ + if (migrate_use_xbzrle()) { + qemu_mutex_unlock(&XBZRLE.lock); + } +} + +/** + * xbzrle_cache_resize: resize the xbzrle cache + * + * This function is called from migrate_params_apply in main + * thread, possibly while a migration is in progress. A running + * migration may be using the cache and might finish during this call, + * hence changes to the cache are protected by XBZRLE.lock(). + * + * Returns 0 for success or -1 for error + * + * @new_size: new cache size + * @errp: set *errp if the check failed, with reason + */ +int xbzrle_cache_resize(uint64_t new_size, Error **errp) +{ + PageCache *new_cache; + int64_t ret = 0; + + /* Check for truncation */ + if (new_size != (size_t)new_size) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size", + "exceeding address space"); + return -1; + } + + if (new_size == migrate_xbzrle_cache_size()) { + /* nothing to do */ + return 0; + } + + XBZRLE_cache_lock(); + + if (XBZRLE.cache != NULL) { + new_cache = cache_init(new_size, TARGET_PAGE_SIZE, errp); + if (!new_cache) { + ret = -1; + goto out; + } + + cache_fini(XBZRLE.cache); + XBZRLE.cache = new_cache; + } +out: + XBZRLE_cache_unlock(); + return ret; +} + +bool ramblock_is_ignored(RAMBlock *block) +{ + return !qemu_ram_is_migratable(block) || + (migrate_ignore_shared() && qemu_ram_is_shared(block)); +} + +#undef RAMBLOCK_FOREACH + +int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque) +{ + RAMBlock *block; + int ret = 0; + + RCU_READ_LOCK_GUARD(); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + ret = func(block, opaque); + if (ret) { + break; + } + } + return ret; +} + +static void ramblock_recv_map_init(void) +{ + RAMBlock *rb; + + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { + assert(!rb->receivedmap); + rb->receivedmap = bitmap_new(rb->max_length >> qemu_target_page_bits()); + } +} + +int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr) +{ + return test_bit(ramblock_recv_bitmap_offset(host_addr, rb), + rb->receivedmap); +} + +bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset) +{ + return test_bit(byte_offset >> TARGET_PAGE_BITS, rb->receivedmap); +} + +void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr) +{ + set_bit_atomic(ramblock_recv_bitmap_offset(host_addr, rb), rb->receivedmap); +} + +void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr, + size_t nr) +{ + bitmap_set_atomic(rb->receivedmap, + ramblock_recv_bitmap_offset(host_addr, rb), + nr); +} + +#define RAMBLOCK_RECV_BITMAP_ENDING (0x0123456789abcdefULL) + +/* + * Format: bitmap_size (8 bytes) + whole_bitmap (N bytes). + * + * Returns >0 if success with sent bytes, or <0 if error. + */ +int64_t ramblock_recv_bitmap_send(QEMUFile *file, + const char *block_name) +{ + RAMBlock *block = qemu_ram_block_by_name(block_name); + unsigned long *le_bitmap, nbits; + uint64_t size; + + if (!block) { + error_report("%s: invalid block name: %s", __func__, block_name); + return -1; + } + + nbits = block->postcopy_length >> TARGET_PAGE_BITS; + + /* + * Make sure the tmp bitmap buffer is big enough, e.g., on 32bit + * machines we may need 4 more bytes for padding (see below + * comment). So extend it a bit before hand. + */ + le_bitmap = bitmap_new(nbits + BITS_PER_LONG); + + /* + * Always use little endian when sending the bitmap. This is + * required that when source and destination VMs are not using the + * same endianness. (Note: big endian won't work.) + */ + bitmap_to_le(le_bitmap, block->receivedmap, nbits); + + /* Size of the bitmap, in bytes */ + size = DIV_ROUND_UP(nbits, 8); + + /* + * size is always aligned to 8 bytes for 64bit machines, but it + * may not be true for 32bit machines. We need this padding to + * make sure the migration can survive even between 32bit and + * 64bit machines. + */ + size = ROUND_UP(size, 8); + + qemu_put_be64(file, size); + qemu_put_buffer(file, (const uint8_t *)le_bitmap, size); + /* + * Mark as an end, in case the middle part is screwed up due to + * some "mysterious" reason. + */ + qemu_put_be64(file, RAMBLOCK_RECV_BITMAP_ENDING); + qemu_fflush(file); + + g_free(le_bitmap); + + if (qemu_file_get_error(file)) { + return qemu_file_get_error(file); + } + + return size + sizeof(size); +} + +/* + * An outstanding page request, on the source, having been received + * and queued + */ +struct RAMSrcPageRequest { + RAMBlock *rb; + hwaddr offset; + hwaddr len; + + QSIMPLEQ_ENTRY(RAMSrcPageRequest) next_req; +}; + +/* State of RAM for migration */ +struct RAMState { + /* QEMUFile used for this migration */ + QEMUFile *f; + /* UFFD file descriptor, used in 'write-tracking' migration */ + int uffdio_fd; + /* Last block that we have visited searching for dirty pages */ + RAMBlock *last_seen_block; + /* Last block from where we have sent data */ + RAMBlock *last_sent_block; + /* Last dirty target page we have sent */ + ram_addr_t last_page; + /* last ram version we have seen */ + uint32_t last_version; + /* How many times we have dirty too many pages */ + int dirty_rate_high_cnt; + /* these variables are used for bitmap sync */ + /* last time we did a full bitmap_sync */ + int64_t time_last_bitmap_sync; + /* bytes transferred at start_time */ + uint64_t bytes_xfer_prev; + /* number of dirty pages since start_time */ + uint64_t num_dirty_pages_period; + /* xbzrle misses since the beginning of the period */ + uint64_t xbzrle_cache_miss_prev; + /* Amount of xbzrle pages since the beginning of the period */ + uint64_t xbzrle_pages_prev; + /* Amount of xbzrle encoded bytes since the beginning of the period */ + uint64_t xbzrle_bytes_prev; + /* Start using XBZRLE (e.g., after the first round). */ + bool xbzrle_enabled; + + /* compression statistics since the beginning of the period */ + /* amount of count that no free thread to compress data */ + uint64_t compress_thread_busy_prev; + /* amount bytes after compression */ + uint64_t compressed_size_prev; + /* amount of compressed pages */ + uint64_t compress_pages_prev; + + /* total handled target pages at the beginning of period */ + uint64_t target_page_count_prev; + /* total handled target pages since start */ + uint64_t target_page_count; + /* number of dirty bits in the bitmap */ + uint64_t migration_dirty_pages; + /* Protects modification of the bitmap and migration dirty pages */ + QemuMutex bitmap_mutex; + /* The RAMBlock used in the last src_page_requests */ + RAMBlock *last_req_rb; + /* Queue of outstanding page requests from the destination */ + QemuMutex src_page_req_mutex; + QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests; +}; +typedef struct RAMState RAMState; + +static RAMState *ram_state; + +static NotifierWithReturnList precopy_notifier_list; + +void precopy_infrastructure_init(void) +{ + notifier_with_return_list_init(&precopy_notifier_list); +} + +void precopy_add_notifier(NotifierWithReturn *n) +{ + notifier_with_return_list_add(&precopy_notifier_list, n); +} + +void precopy_remove_notifier(NotifierWithReturn *n) +{ + notifier_with_return_remove(n); +} + +int precopy_notify(PrecopyNotifyReason reason, Error **errp) +{ + PrecopyNotifyData pnd; + pnd.reason = reason; + pnd.errp = errp; + + return notifier_with_return_list_notify(&precopy_notifier_list, &pnd); +} + +uint64_t ram_bytes_remaining(void) +{ + return ram_state ? (ram_state->migration_dirty_pages * TARGET_PAGE_SIZE) : + 0; +} + +MigrationStats ram_counters; + +/* used by the search for pages to send */ +struct PageSearchStatus { + /* Current block being searched */ + RAMBlock *block; + /* Current page to search from */ + unsigned long page; + /* Set once we wrap around */ + bool complete_round; +}; +typedef struct PageSearchStatus PageSearchStatus; + +CompressionStats compression_counters; + +struct CompressParam { + bool done; + bool quit; + bool zero_page; + QEMUFile *file; + QemuMutex mutex; + QemuCond cond; + RAMBlock *block; + ram_addr_t offset; + + /* internally used fields */ + z_stream stream; + uint8_t *originbuf; +}; +typedef struct CompressParam CompressParam; + +struct DecompressParam { + bool done; + bool quit; + QemuMutex mutex; + QemuCond cond; + void *des; + uint8_t *compbuf; + int len; + z_stream stream; +}; +typedef struct DecompressParam DecompressParam; + +static CompressParam *comp_param; +static QemuThread *compress_threads; +/* comp_done_cond is used to wake up the migration thread when + * one of the compression threads has finished the compression. + * comp_done_lock is used to co-work with comp_done_cond. + */ +static QemuMutex comp_done_lock; +static QemuCond comp_done_cond; +/* The empty QEMUFileOps will be used by file in CompressParam */ +static const QEMUFileOps empty_ops = { }; + +static QEMUFile *decomp_file; +static DecompressParam *decomp_param; +static QemuThread *decompress_threads; +static QemuMutex decomp_done_lock; +static QemuCond decomp_done_cond; + +static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block, + ram_addr_t offset, uint8_t *source_buf); + +static void *do_data_compress(void *opaque) +{ + CompressParam *param = opaque; + RAMBlock *block; + ram_addr_t offset; + bool zero_page; + + qemu_mutex_lock(¶m->mutex); + while (!param->quit) { + if (param->block) { + block = param->block; + offset = param->offset; + param->block = NULL; + qemu_mutex_unlock(¶m->mutex); + + zero_page = do_compress_ram_page(param->file, ¶m->stream, + block, offset, param->originbuf); + + qemu_mutex_lock(&comp_done_lock); + param->done = true; + param->zero_page = zero_page; + qemu_cond_signal(&comp_done_cond); + qemu_mutex_unlock(&comp_done_lock); + + qemu_mutex_lock(¶m->mutex); + } else { + qemu_cond_wait(¶m->cond, ¶m->mutex); + } + } + qemu_mutex_unlock(¶m->mutex); + + return NULL; +} + +static void compress_threads_save_cleanup(void) +{ + int i, thread_count; + + if (!migrate_use_compression() || !comp_param) { + return; + } + + thread_count = migrate_compress_threads(); + for (i = 0; i < thread_count; i++) { + /* + * we use it as a indicator which shows if the thread is + * properly init'd or not + */ + if (!comp_param[i].file) { + break; + } + + qemu_mutex_lock(&comp_param[i].mutex); + comp_param[i].quit = true; + qemu_cond_signal(&comp_param[i].cond); + qemu_mutex_unlock(&comp_param[i].mutex); + + qemu_thread_join(compress_threads + i); + qemu_mutex_destroy(&comp_param[i].mutex); + qemu_cond_destroy(&comp_param[i].cond); + deflateEnd(&comp_param[i].stream); + g_free(comp_param[i].originbuf); + qemu_fclose(comp_param[i].file); + comp_param[i].file = NULL; + } + qemu_mutex_destroy(&comp_done_lock); + qemu_cond_destroy(&comp_done_cond); + g_free(compress_threads); + g_free(comp_param); + compress_threads = NULL; + comp_param = NULL; +} + +static int compress_threads_save_setup(void) +{ + int i, thread_count; + + if (!migrate_use_compression()) { + return 0; + } + thread_count = migrate_compress_threads(); + compress_threads = g_new0(QemuThread, thread_count); + comp_param = g_new0(CompressParam, thread_count); + qemu_cond_init(&comp_done_cond); + qemu_mutex_init(&comp_done_lock); + for (i = 0; i < thread_count; i++) { + comp_param[i].originbuf = g_try_malloc(TARGET_PAGE_SIZE); + if (!comp_param[i].originbuf) { + goto exit; + } + + if (deflateInit(&comp_param[i].stream, + migrate_compress_level()) != Z_OK) { + g_free(comp_param[i].originbuf); + goto exit; + } + + /* comp_param[i].file is just used as a dummy buffer to save data, + * set its ops to empty. + */ + comp_param[i].file = qemu_fopen_ops(NULL, &empty_ops, false); + comp_param[i].done = true; + comp_param[i].quit = false; + qemu_mutex_init(&comp_param[i].mutex); + qemu_cond_init(&comp_param[i].cond); + qemu_thread_create(compress_threads + i, "compress", + do_data_compress, comp_param + i, + QEMU_THREAD_JOINABLE); + } + return 0; + +exit: + compress_threads_save_cleanup(); + return -1; +} + +/** + * save_page_header: write page header to wire + * + * If this is the 1st block, it also writes the block identification + * + * Returns the number of bytes written + * + * @f: QEMUFile where to send the data + * @block: block that contains the page we want to send + * @offset: offset inside the block for the page + * in the lower bits, it contains flags + */ +static size_t save_page_header(RAMState *rs, QEMUFile *f, RAMBlock *block, + ram_addr_t offset) +{ + size_t size, len; + + if (block == rs->last_sent_block) { + offset |= RAM_SAVE_FLAG_CONTINUE; + } + qemu_put_be64(f, offset); + size = 8; + + if (!(offset & RAM_SAVE_FLAG_CONTINUE)) { + len = strlen(block->idstr); + qemu_put_byte(f, len); + qemu_put_buffer(f, (uint8_t *)block->idstr, len); + size += 1 + len; + rs->last_sent_block = block; + } + return size; +} + +/** + * mig_throttle_guest_down: throttle down the guest + * + * Reduce amount of guest cpu execution to hopefully slow down memory + * writes. If guest dirty memory rate is reduced below the rate at + * which we can transfer pages to the destination then we should be + * able to complete migration. Some workloads dirty memory way too + * fast and will not effectively converge, even with auto-converge. + */ +static void mig_throttle_guest_down(uint64_t bytes_dirty_period, + uint64_t bytes_dirty_threshold) +{ + MigrationState *s = migrate_get_current(); + uint64_t pct_initial = s->parameters.cpu_throttle_initial; + uint64_t pct_increment = s->parameters.cpu_throttle_increment; + bool pct_tailslow = s->parameters.cpu_throttle_tailslow; + int pct_max = s->parameters.max_cpu_throttle; + + uint64_t throttle_now = cpu_throttle_get_percentage(); + uint64_t cpu_now, cpu_ideal, throttle_inc; + + /* We have not started throttling yet. Let's start it. */ + if (!cpu_throttle_active()) { + cpu_throttle_set(pct_initial); + } else { + /* Throttling already on, just increase the rate */ + if (!pct_tailslow) { + throttle_inc = pct_increment; + } else { + /* Compute the ideal CPU percentage used by Guest, which may + * make the dirty rate match the dirty rate threshold. */ + cpu_now = 100 - throttle_now; + cpu_ideal = cpu_now * (bytes_dirty_threshold * 1.0 / + bytes_dirty_period); + throttle_inc = MIN(cpu_now - cpu_ideal, pct_increment); + } + cpu_throttle_set(MIN(throttle_now + throttle_inc, pct_max)); + } +} + +void mig_throttle_counter_reset(void) +{ + RAMState *rs = ram_state; + + rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + rs->num_dirty_pages_period = 0; + rs->bytes_xfer_prev = ram_counters.transferred; +} + +/** + * xbzrle_cache_zero_page: insert a zero page in the XBZRLE cache + * + * @rs: current RAM state + * @current_addr: address for the zero page + * + * Update the xbzrle cache to reflect a page that's been sent as all 0. + * The important thing is that a stale (not-yet-0'd) page be replaced + * by the new data. + * As a bonus, if the page wasn't in the cache it gets added so that + * when a small write is made into the 0'd page it gets XBZRLE sent. + */ +static void xbzrle_cache_zero_page(RAMState *rs, ram_addr_t current_addr) +{ + if (!rs->xbzrle_enabled) { + return; + } + + /* We don't care if this fails to allocate a new cache page + * as long as it updated an old one */ + cache_insert(XBZRLE.cache, current_addr, XBZRLE.zero_target_page, + ram_counters.dirty_sync_count); +} + +#define ENCODING_FLAG_XBZRLE 0x1 + +/** + * save_xbzrle_page: compress and send current page + * + * Returns: 1 means that we wrote the page + * 0 means that page is identical to the one already sent + * -1 means that xbzrle would be longer than normal + * + * @rs: current RAM state + * @current_data: pointer to the address of the page contents + * @current_addr: addr of the page + * @block: block that contains the page we want to send + * @offset: offset inside the block for the page + * @last_stage: if we are at the completion stage + */ +static int save_xbzrle_page(RAMState *rs, uint8_t **current_data, + ram_addr_t current_addr, RAMBlock *block, + ram_addr_t offset, bool last_stage) +{ + int encoded_len = 0, bytes_xbzrle; + uint8_t *prev_cached_page; + + if (!cache_is_cached(XBZRLE.cache, current_addr, + ram_counters.dirty_sync_count)) { + xbzrle_counters.cache_miss++; + if (!last_stage) { + if (cache_insert(XBZRLE.cache, current_addr, *current_data, + ram_counters.dirty_sync_count) == -1) { + return -1; + } else { + /* update *current_data when the page has been + inserted into cache */ + *current_data = get_cached_data(XBZRLE.cache, current_addr); + } + } + return -1; + } + + /* + * Reaching here means the page has hit the xbzrle cache, no matter what + * encoding result it is (normal encoding, overflow or skipping the page), + * count the page as encoded. This is used to calculate the encoding rate. + * + * Example: 2 pages (8KB) being encoded, first page encoding generates 2KB, + * 2nd page turns out to be skipped (i.e. no new bytes written to the + * page), the overall encoding rate will be 8KB / 2KB = 4, which has the + * skipped page included. In this way, the encoding rate can tell if the + * guest page is good for xbzrle encoding. + */ + xbzrle_counters.pages++; + prev_cached_page = get_cached_data(XBZRLE.cache, current_addr); + + /* save current buffer into memory */ + memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE); + + /* XBZRLE encoding (if there is no overflow) */ + encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf, + TARGET_PAGE_SIZE, XBZRLE.encoded_buf, + TARGET_PAGE_SIZE); + + /* + * Update the cache contents, so that it corresponds to the data + * sent, in all cases except where we skip the page. + */ + if (!last_stage && encoded_len != 0) { + memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE); + /* + * In the case where we couldn't compress, ensure that the caller + * sends the data from the cache, since the guest might have + * changed the RAM since we copied it. + */ + *current_data = prev_cached_page; + } + + if (encoded_len == 0) { + trace_save_xbzrle_page_skipping(); + return 0; + } else if (encoded_len == -1) { + trace_save_xbzrle_page_overflow(); + xbzrle_counters.overflow++; + xbzrle_counters.bytes += TARGET_PAGE_SIZE; + return -1; + } + + /* Send XBZRLE based compressed page */ + bytes_xbzrle = save_page_header(rs, rs->f, block, + offset | RAM_SAVE_FLAG_XBZRLE); + qemu_put_byte(rs->f, ENCODING_FLAG_XBZRLE); + qemu_put_be16(rs->f, encoded_len); + qemu_put_buffer(rs->f, XBZRLE.encoded_buf, encoded_len); + bytes_xbzrle += encoded_len + 1 + 2; + /* + * Like compressed_size (please see update_compress_thread_counts), + * the xbzrle encoded bytes don't count the 8 byte header with + * RAM_SAVE_FLAG_CONTINUE. + */ + xbzrle_counters.bytes += bytes_xbzrle - 8; + ram_counters.transferred += bytes_xbzrle; + + return 1; +} + +/** + * migration_bitmap_find_dirty: find the next dirty page from start + * + * Returns the page offset within memory region of the start of a dirty page + * + * @rs: current RAM state + * @rb: RAMBlock where to search for dirty pages + * @start: page where we start the search + */ +static inline +unsigned long migration_bitmap_find_dirty(RAMState *rs, RAMBlock *rb, + unsigned long start) +{ + unsigned long size = rb->used_length >> TARGET_PAGE_BITS; + unsigned long *bitmap = rb->bmap; + + if (ramblock_is_ignored(rb)) { + return size; + } + + return find_next_bit(bitmap, size, start); +} + +static void migration_clear_memory_region_dirty_bitmap(RAMBlock *rb, + unsigned long page) +{ + uint8_t shift; + hwaddr size, start; + + if (!rb->clear_bmap || !clear_bmap_test_and_clear(rb, page)) { + return; + } + + shift = rb->clear_bmap_shift; + /* + * CLEAR_BITMAP_SHIFT_MIN should always guarantee this... this + * can make things easier sometimes since then start address + * of the small chunk will always be 64 pages aligned so the + * bitmap will always be aligned to unsigned long. We should + * even be able to remove this restriction but I'm simply + * keeping it. + */ + assert(shift >= 6); + + size = 1ULL << (TARGET_PAGE_BITS + shift); + start = QEMU_ALIGN_DOWN((ram_addr_t)page << TARGET_PAGE_BITS, size); + trace_migration_bitmap_clear_dirty(rb->idstr, start, size, page); + memory_region_clear_dirty_bitmap(rb->mr, start, size); +} + +static void +migration_clear_memory_region_dirty_bitmap_range(RAMBlock *rb, + unsigned long start, + unsigned long npages) +{ + unsigned long i, chunk_pages = 1UL << rb->clear_bmap_shift; + unsigned long chunk_start = QEMU_ALIGN_DOWN(start, chunk_pages); + unsigned long chunk_end = QEMU_ALIGN_UP(start + npages, chunk_pages); + + /* + * Clear pages from start to start + npages - 1, so the end boundary is + * exclusive. + */ + for (i = chunk_start; i < chunk_end; i += chunk_pages) { + migration_clear_memory_region_dirty_bitmap(rb, i); + } +} + +/* + * colo_bitmap_find_diry:find contiguous dirty pages from start + * + * Returns the page offset within memory region of the start of the contiguout + * dirty page + * + * @rs: current RAM state + * @rb: RAMBlock where to search for dirty pages + * @start: page where we start the search + * @num: the number of contiguous dirty pages + */ +static inline +unsigned long colo_bitmap_find_dirty(RAMState *rs, RAMBlock *rb, + unsigned long start, unsigned long *num) +{ + unsigned long size = rb->used_length >> TARGET_PAGE_BITS; + unsigned long *bitmap = rb->bmap; + unsigned long first, next; + + *num = 0; + + if (ramblock_is_ignored(rb)) { + return size; + } + + first = find_next_bit(bitmap, size, start); + if (first >= size) { + return first; + } + next = find_next_zero_bit(bitmap, size, first + 1); + assert(next >= first); + *num = next - first; + return first; +} + +static inline bool migration_bitmap_clear_dirty(RAMState *rs, + RAMBlock *rb, + unsigned long page) +{ + bool ret; + + /* + * Clear dirty bitmap if needed. This _must_ be called before we + * send any of the page in the chunk because we need to make sure + * we can capture further page content changes when we sync dirty + * log the next time. So as long as we are going to send any of + * the page in the chunk we clear the remote dirty bitmap for all. + * Clearing it earlier won't be a problem, but too late will. + */ + migration_clear_memory_region_dirty_bitmap(rb, page); + + ret = test_and_clear_bit(page, rb->bmap); + if (ret) { + rs->migration_dirty_pages--; + } + + return ret; +} + +static void dirty_bitmap_clear_section(MemoryRegionSection *section, + void *opaque) +{ + const hwaddr offset = section->offset_within_region; + const hwaddr size = int128_get64(section->size); + const unsigned long start = offset >> TARGET_PAGE_BITS; + const unsigned long npages = size >> TARGET_PAGE_BITS; + RAMBlock *rb = section->mr->ram_block; + uint64_t *cleared_bits = opaque; + + /* + * We don't grab ram_state->bitmap_mutex because we expect to run + * only when starting migration or during postcopy recovery where + * we don't have concurrent access. + */ + if (!migration_in_postcopy() && !migrate_background_snapshot()) { + migration_clear_memory_region_dirty_bitmap_range(rb, start, npages); + } + *cleared_bits += bitmap_count_one_with_offset(rb->bmap, start, npages); + bitmap_clear(rb->bmap, start, npages); +} + +/* + * Exclude all dirty pages from migration that fall into a discarded range as + * managed by a RamDiscardManager responsible for the mapped memory region of + * the RAMBlock. Clear the corresponding bits in the dirty bitmaps. + * + * Discarded pages ("logically unplugged") have undefined content and must + * not get migrated, because even reading these pages for migration might + * result in undesired behavior. + * + * Returns the number of cleared bits in the RAMBlock dirty bitmap. + * + * Note: The result is only stable while migrating (precopy/postcopy). + */ +static uint64_t ramblock_dirty_bitmap_clear_discarded_pages(RAMBlock *rb) +{ + uint64_t cleared_bits = 0; + + if (rb->mr && rb->bmap && memory_region_has_ram_discard_manager(rb->mr)) { + RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); + MemoryRegionSection section = { + .mr = rb->mr, + .offset_within_region = 0, + .size = int128_make64(qemu_ram_get_used_length(rb)), + }; + + ram_discard_manager_replay_discarded(rdm, §ion, + dirty_bitmap_clear_section, + &cleared_bits); + } + return cleared_bits; +} + +/* + * Check if a host-page aligned page falls into a discarded range as managed by + * a RamDiscardManager responsible for the mapped memory region of the RAMBlock. + * + * Note: The result is only stable while migrating (precopy/postcopy). + */ +bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start) +{ + if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) { + RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); + MemoryRegionSection section = { + .mr = rb->mr, + .offset_within_region = start, + .size = int128_make64(qemu_ram_pagesize(rb)), + }; + + return !ram_discard_manager_is_populated(rdm, §ion); + } + return false; +} + +/* Called with RCU critical section */ +static void ramblock_sync_dirty_bitmap(RAMState *rs, RAMBlock *rb) +{ + uint64_t new_dirty_pages = + cpu_physical_memory_sync_dirty_bitmap(rb, 0, rb->used_length); + + rs->migration_dirty_pages += new_dirty_pages; + rs->num_dirty_pages_period += new_dirty_pages; +} + +/** + * ram_pagesize_summary: calculate all the pagesizes of a VM + * + * Returns a summary bitmap of the page sizes of all RAMBlocks + * + * For VMs with just normal pages this is equivalent to the host page + * size. If it's got some huge pages then it's the OR of all the + * different page sizes. + */ +uint64_t ram_pagesize_summary(void) +{ + RAMBlock *block; + uint64_t summary = 0; + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + summary |= block->page_size; + } + + return summary; +} + +uint64_t ram_get_total_transferred_pages(void) +{ + return ram_counters.normal + ram_counters.duplicate + + compression_counters.pages + xbzrle_counters.pages; +} + +static void migration_update_rates(RAMState *rs, int64_t end_time) +{ + uint64_t page_count = rs->target_page_count - rs->target_page_count_prev; + double compressed_size; + + /* calculate period counters */ + ram_counters.dirty_pages_rate = rs->num_dirty_pages_period * 1000 + / (end_time - rs->time_last_bitmap_sync); + + if (!page_count) { + return; + } + + if (migrate_use_xbzrle()) { + double encoded_size, unencoded_size; + + xbzrle_counters.cache_miss_rate = (double)(xbzrle_counters.cache_miss - + rs->xbzrle_cache_miss_prev) / page_count; + rs->xbzrle_cache_miss_prev = xbzrle_counters.cache_miss; + unencoded_size = (xbzrle_counters.pages - rs->xbzrle_pages_prev) * + TARGET_PAGE_SIZE; + encoded_size = xbzrle_counters.bytes - rs->xbzrle_bytes_prev; + if (xbzrle_counters.pages == rs->xbzrle_pages_prev || !encoded_size) { + xbzrle_counters.encoding_rate = 0; + } else { + xbzrle_counters.encoding_rate = unencoded_size / encoded_size; + } + rs->xbzrle_pages_prev = xbzrle_counters.pages; + rs->xbzrle_bytes_prev = xbzrle_counters.bytes; + } + + if (migrate_use_compression()) { + compression_counters.busy_rate = (double)(compression_counters.busy - + rs->compress_thread_busy_prev) / page_count; + rs->compress_thread_busy_prev = compression_counters.busy; + + compressed_size = compression_counters.compressed_size - + rs->compressed_size_prev; + if (compressed_size) { + double uncompressed_size = (compression_counters.pages - + rs->compress_pages_prev) * TARGET_PAGE_SIZE; + + /* Compression-Ratio = Uncompressed-size / Compressed-size */ + compression_counters.compression_rate = + uncompressed_size / compressed_size; + + rs->compress_pages_prev = compression_counters.pages; + rs->compressed_size_prev = compression_counters.compressed_size; + } + } +} + +static void migration_trigger_throttle(RAMState *rs) +{ + MigrationState *s = migrate_get_current(); + uint64_t threshold = s->parameters.throttle_trigger_threshold; + + uint64_t bytes_xfer_period = ram_counters.transferred - rs->bytes_xfer_prev; + uint64_t bytes_dirty_period = rs->num_dirty_pages_period * TARGET_PAGE_SIZE; + uint64_t bytes_dirty_threshold = bytes_xfer_period * threshold / 100; + + /* During block migration the auto-converge logic incorrectly detects + * that ram migration makes no progress. Avoid this by disabling the + * throttling logic during the bulk phase of block migration. */ + if (migrate_auto_converge() && !blk_mig_bulk_active()) { + /* The following detection logic can be refined later. For now: + Check to see if the ratio between dirtied bytes and the approx. + amount of bytes that just got transferred since the last time + we were in this routine reaches the threshold. If that happens + twice, start or increase throttling. */ + + if ((bytes_dirty_period > bytes_dirty_threshold) && + (++rs->dirty_rate_high_cnt >= 2)) { + trace_migration_throttle(); + rs->dirty_rate_high_cnt = 0; + mig_throttle_guest_down(bytes_dirty_period, + bytes_dirty_threshold); + } + } +} + +static void migration_bitmap_sync(RAMState *rs) +{ + RAMBlock *block; + int64_t end_time; + + ram_counters.dirty_sync_count++; + + if (!rs->time_last_bitmap_sync) { + rs->time_last_bitmap_sync = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + } + + trace_migration_bitmap_sync_start(); + memory_global_dirty_log_sync(); + + qemu_mutex_lock(&rs->bitmap_mutex); + WITH_RCU_READ_LOCK_GUARD() { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + ramblock_sync_dirty_bitmap(rs, block); + } + ram_counters.remaining = ram_bytes_remaining(); + } + qemu_mutex_unlock(&rs->bitmap_mutex); + + memory_global_after_dirty_log_sync(); + trace_migration_bitmap_sync_end(rs->num_dirty_pages_period); + + end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + + /* more than 1 second = 1000 millisecons */ + if (end_time > rs->time_last_bitmap_sync + 1000) { + migration_trigger_throttle(rs); + + migration_update_rates(rs, end_time); + + rs->target_page_count_prev = rs->target_page_count; + + /* reset period counters */ + rs->time_last_bitmap_sync = end_time; + rs->num_dirty_pages_period = 0; + rs->bytes_xfer_prev = ram_counters.transferred; + } + if (migrate_use_events()) { + qapi_event_send_migration_pass(ram_counters.dirty_sync_count); + } +} + +static void migration_bitmap_sync_precopy(RAMState *rs) +{ + Error *local_err = NULL; + + /* + * The current notifier usage is just an optimization to migration, so we + * don't stop the normal migration process in the error case. + */ + if (precopy_notify(PRECOPY_NOTIFY_BEFORE_BITMAP_SYNC, &local_err)) { + error_report_err(local_err); + local_err = NULL; + } + + migration_bitmap_sync(rs); + + if (precopy_notify(PRECOPY_NOTIFY_AFTER_BITMAP_SYNC, &local_err)) { + error_report_err(local_err); + } +} + +/** + * save_zero_page_to_file: send the zero page to the file + * + * Returns the size of data written to the file, 0 means the page is not + * a zero page + * + * @rs: current RAM state + * @file: the file where the data is saved + * @block: block that contains the page we want to send + * @offset: offset inside the block for the page + */ +static int save_zero_page_to_file(RAMState *rs, QEMUFile *file, + RAMBlock *block, ram_addr_t offset) +{ + uint8_t *p = block->host + offset; + int len = 0; + + if (is_zero_range(p, TARGET_PAGE_SIZE)) { + len += save_page_header(rs, file, block, offset | RAM_SAVE_FLAG_ZERO); + qemu_put_byte(file, 0); + len += 1; + } + return len; +} + +/** + * save_zero_page: send the zero page to the stream + * + * Returns the number of pages written. + * + * @rs: current RAM state + * @block: block that contains the page we want to send + * @offset: offset inside the block for the page + */ +static int save_zero_page(RAMState *rs, RAMBlock *block, ram_addr_t offset) +{ + int len = save_zero_page_to_file(rs, rs->f, block, offset); + + if (len) { + ram_counters.duplicate++; + ram_counters.transferred += len; + return 1; + } + return -1; +} + +static void ram_release_pages(const char *rbname, uint64_t offset, int pages) +{ + if (!migrate_release_ram() || !migration_in_postcopy()) { + return; + } + + ram_discard_range(rbname, offset, ((ram_addr_t)pages) << TARGET_PAGE_BITS); +} + +/* + * @pages: the number of pages written by the control path, + * < 0 - error + * > 0 - number of pages written + * + * Return true if the pages has been saved, otherwise false is returned. + */ +static bool control_save_page(RAMState *rs, RAMBlock *block, ram_addr_t offset, + int *pages) +{ + uint64_t bytes_xmit = 0; + int ret; + + *pages = -1; + ret = ram_control_save_page(rs->f, block->offset, offset, TARGET_PAGE_SIZE, + &bytes_xmit); + if (ret == RAM_SAVE_CONTROL_NOT_SUPP) { + return false; + } + + if (bytes_xmit) { + ram_counters.transferred += bytes_xmit; + *pages = 1; + } + + if (ret == RAM_SAVE_CONTROL_DELAYED) { + return true; + } + + if (bytes_xmit > 0) { + ram_counters.normal++; + } else if (bytes_xmit == 0) { + ram_counters.duplicate++; + } + + return true; +} + +/* + * directly send the page to the stream + * + * Returns the number of pages written. + * + * @rs: current RAM state + * @block: block that contains the page we want to send + * @offset: offset inside the block for the page + * @buf: the page to be sent + * @async: send to page asyncly + */ +static int save_normal_page(RAMState *rs, RAMBlock *block, ram_addr_t offset, + uint8_t *buf, bool async) +{ + ram_counters.transferred += save_page_header(rs, rs->f, block, + offset | RAM_SAVE_FLAG_PAGE); + if (async) { + qemu_put_buffer_async(rs->f, buf, TARGET_PAGE_SIZE, + migrate_release_ram() & + migration_in_postcopy()); + } else { + qemu_put_buffer(rs->f, buf, TARGET_PAGE_SIZE); + } + ram_counters.transferred += TARGET_PAGE_SIZE; + ram_counters.normal++; + return 1; +} + +/** + * ram_save_page: send the given page to the stream + * + * Returns the number of pages written. + * < 0 - error + * >=0 - Number of pages written - this might legally be 0 + * if xbzrle noticed the page was the same. + * + * @rs: current RAM state + * @block: block that contains the page we want to send + * @offset: offset inside the block for the page + * @last_stage: if we are at the completion stage + */ +static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage) +{ + int pages = -1; + uint8_t *p; + bool send_async = true; + RAMBlock *block = pss->block; + ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS; + ram_addr_t current_addr = block->offset + offset; + + p = block->host + offset; + trace_ram_save_page(block->idstr, (uint64_t)offset, p); + + XBZRLE_cache_lock(); + if (rs->xbzrle_enabled && !migration_in_postcopy()) { + pages = save_xbzrle_page(rs, &p, current_addr, block, + offset, last_stage); + if (!last_stage) { + /* Can't send this cached data async, since the cache page + * might get updated before it gets to the wire + */ + send_async = false; + } + } + + /* XBZRLE overflow or normal page */ + if (pages == -1) { + pages = save_normal_page(rs, block, offset, p, send_async); + } + + XBZRLE_cache_unlock(); + + return pages; +} + +static int ram_save_multifd_page(RAMState *rs, RAMBlock *block, + ram_addr_t offset) +{ + if (multifd_queue_page(rs->f, block, offset) < 0) { + return -1; + } + ram_counters.normal++; + + return 1; +} + +static bool do_compress_ram_page(QEMUFile *f, z_stream *stream, RAMBlock *block, + ram_addr_t offset, uint8_t *source_buf) +{ + RAMState *rs = ram_state; + uint8_t *p = block->host + (offset & TARGET_PAGE_MASK); + bool zero_page = false; + int ret; + + if (save_zero_page_to_file(rs, f, block, offset)) { + zero_page = true; + goto exit; + } + + save_page_header(rs, f, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE); + + /* + * copy it to a internal buffer to avoid it being modified by VM + * so that we can catch up the error during compression and + * decompression + */ + memcpy(source_buf, p, TARGET_PAGE_SIZE); + ret = qemu_put_compression_data(f, stream, source_buf, TARGET_PAGE_SIZE); + if (ret < 0) { + qemu_file_set_error(migrate_get_current()->to_dst_file, ret); + error_report("compressed data failed!"); + return false; + } + +exit: + ram_release_pages(block->idstr, offset & TARGET_PAGE_MASK, 1); + return zero_page; +} + +static void +update_compress_thread_counts(const CompressParam *param, int bytes_xmit) +{ + ram_counters.transferred += bytes_xmit; + + if (param->zero_page) { + ram_counters.duplicate++; + return; + } + + /* 8 means a header with RAM_SAVE_FLAG_CONTINUE. */ + compression_counters.compressed_size += bytes_xmit - 8; + compression_counters.pages++; +} + +static bool save_page_use_compression(RAMState *rs); + +static void flush_compressed_data(RAMState *rs) +{ + int idx, len, thread_count; + + if (!save_page_use_compression(rs)) { + return; + } + thread_count = migrate_compress_threads(); + + qemu_mutex_lock(&comp_done_lock); + for (idx = 0; idx < thread_count; idx++) { + while (!comp_param[idx].done) { + qemu_cond_wait(&comp_done_cond, &comp_done_lock); + } + } + qemu_mutex_unlock(&comp_done_lock); + + for (idx = 0; idx < thread_count; idx++) { + qemu_mutex_lock(&comp_param[idx].mutex); + if (!comp_param[idx].quit) { + len = qemu_put_qemu_file(rs->f, comp_param[idx].file); + /* + * it's safe to fetch zero_page without holding comp_done_lock + * as there is no further request submitted to the thread, + * i.e, the thread should be waiting for a request at this point. + */ + update_compress_thread_counts(&comp_param[idx], len); + } + qemu_mutex_unlock(&comp_param[idx].mutex); + } +} + +static inline void set_compress_params(CompressParam *param, RAMBlock *block, + ram_addr_t offset) +{ + param->block = block; + param->offset = offset; +} + +static int compress_page_with_multi_thread(RAMState *rs, RAMBlock *block, + ram_addr_t offset) +{ + int idx, thread_count, bytes_xmit = -1, pages = -1; + bool wait = migrate_compress_wait_thread(); + + thread_count = migrate_compress_threads(); + qemu_mutex_lock(&comp_done_lock); +retry: + for (idx = 0; idx < thread_count; idx++) { + if (comp_param[idx].done) { + comp_param[idx].done = false; + bytes_xmit = qemu_put_qemu_file(rs->f, comp_param[idx].file); + qemu_mutex_lock(&comp_param[idx].mutex); + set_compress_params(&comp_param[idx], block, offset); + qemu_cond_signal(&comp_param[idx].cond); + qemu_mutex_unlock(&comp_param[idx].mutex); + pages = 1; + update_compress_thread_counts(&comp_param[idx], bytes_xmit); + break; + } + } + + /* + * wait for the free thread if the user specifies 'compress-wait-thread', + * otherwise we will post the page out in the main thread as normal page. + */ + if (pages < 0 && wait) { + qemu_cond_wait(&comp_done_cond, &comp_done_lock); + goto retry; + } + qemu_mutex_unlock(&comp_done_lock); + + return pages; +} + +/** + * find_dirty_block: find the next dirty page and update any state + * associated with the search process. + * + * Returns true if a page is found + * + * @rs: current RAM state + * @pss: data about the state of the current dirty page scan + * @again: set to false if the search has scanned the whole of RAM + */ +static bool find_dirty_block(RAMState *rs, PageSearchStatus *pss, bool *again) +{ + pss->page = migration_bitmap_find_dirty(rs, pss->block, pss->page); + if (pss->complete_round && pss->block == rs->last_seen_block && + pss->page >= rs->last_page) { + /* + * We've been once around the RAM and haven't found anything. + * Give up. + */ + *again = false; + return false; + } + if (!offset_in_ramblock(pss->block, + ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)) { + /* Didn't find anything in this RAM Block */ + pss->page = 0; + pss->block = QLIST_NEXT_RCU(pss->block, next); + if (!pss->block) { + /* + * If memory migration starts over, we will meet a dirtied page + * which may still exists in compression threads's ring, so we + * should flush the compressed data to make sure the new page + * is not overwritten by the old one in the destination. + * + * Also If xbzrle is on, stop using the data compression at this + * point. In theory, xbzrle can do better than compression. + */ + flush_compressed_data(rs); + + /* Hit the end of the list */ + pss->block = QLIST_FIRST_RCU(&ram_list.blocks); + /* Flag that we've looped */ + pss->complete_round = true; + /* After the first round, enable XBZRLE. */ + if (migrate_use_xbzrle()) { + rs->xbzrle_enabled = true; + } + } + /* Didn't find anything this time, but try again on the new block */ + *again = true; + return false; + } else { + /* Can go around again, but... */ + *again = true; + /* We've found something so probably don't need to */ + return true; + } +} + +/** + * unqueue_page: gets a page of the queue + * + * Helper for 'get_queued_page' - gets a page off the queue + * + * Returns the block of the page (or NULL if none available) + * + * @rs: current RAM state + * @offset: used to return the offset within the RAMBlock + */ +static RAMBlock *unqueue_page(RAMState *rs, ram_addr_t *offset) +{ + RAMBlock *block = NULL; + + if (QSIMPLEQ_EMPTY_ATOMIC(&rs->src_page_requests)) { + return NULL; + } + + QEMU_LOCK_GUARD(&rs->src_page_req_mutex); + if (!QSIMPLEQ_EMPTY(&rs->src_page_requests)) { + struct RAMSrcPageRequest *entry = + QSIMPLEQ_FIRST(&rs->src_page_requests); + block = entry->rb; + *offset = entry->offset; + + if (entry->len > TARGET_PAGE_SIZE) { + entry->len -= TARGET_PAGE_SIZE; + entry->offset += TARGET_PAGE_SIZE; + } else { + memory_region_unref(block->mr); + QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req); + g_free(entry); + migration_consume_urgent_request(); + } + } + + return block; +} + +#if defined(__linux__) +/** + * poll_fault_page: try to get next UFFD write fault page and, if pending fault + * is found, return RAM block pointer and page offset + * + * Returns pointer to the RAMBlock containing faulting page, + * NULL if no write faults are pending + * + * @rs: current RAM state + * @offset: page offset from the beginning of the block + */ +static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset) +{ + struct uffd_msg uffd_msg; + void *page_address; + RAMBlock *block; + int res; + + if (!migrate_background_snapshot()) { + return NULL; + } + + res = uffd_read_events(rs->uffdio_fd, &uffd_msg, 1); + if (res <= 0) { + return NULL; + } + + page_address = (void *)(uintptr_t) uffd_msg.arg.pagefault.address; + block = qemu_ram_block_from_host(page_address, false, offset); + assert(block && (block->flags & RAM_UF_WRITEPROTECT) != 0); + return block; +} + +/** + * ram_save_release_protection: release UFFD write protection after + * a range of pages has been saved + * + * @rs: current RAM state + * @pss: page-search-status structure + * @start_page: index of the first page in the range relative to pss->block + * + * Returns 0 on success, negative value in case of an error +*/ +static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss, + unsigned long start_page) +{ + int res = 0; + + /* Check if page is from UFFD-managed region. */ + if (pss->block->flags & RAM_UF_WRITEPROTECT) { + void *page_address = pss->block->host + (start_page << TARGET_PAGE_BITS); + uint64_t run_length = (pss->page - start_page + 1) << TARGET_PAGE_BITS; + + /* Flush async buffers before un-protect. */ + qemu_fflush(rs->f); + /* Un-protect memory range. */ + res = uffd_change_protection(rs->uffdio_fd, page_address, run_length, + false, false); + } + + return res; +} + +/* ram_write_tracking_available: check if kernel supports required UFFD features + * + * Returns true if supports, false otherwise + */ +bool ram_write_tracking_available(void) +{ + uint64_t uffd_features; + int res; + + res = uffd_query_features(&uffd_features); + return (res == 0 && + (uffd_features & UFFD_FEATURE_PAGEFAULT_FLAG_WP) != 0); +} + +/* ram_write_tracking_compatible: check if guest configuration is + * compatible with 'write-tracking' + * + * Returns true if compatible, false otherwise + */ +bool ram_write_tracking_compatible(void) +{ + const uint64_t uffd_ioctls_mask = BIT(_UFFDIO_WRITEPROTECT); + int uffd_fd; + RAMBlock *block; + bool ret = false; + + /* Open UFFD file descriptor */ + uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, false); + if (uffd_fd < 0) { + return false; + } + + RCU_READ_LOCK_GUARD(); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + uint64_t uffd_ioctls; + + /* Nothing to do with read-only and MMIO-writable regions */ + if (block->mr->readonly || block->mr->rom_device) { + continue; + } + /* Try to register block memory via UFFD-IO to track writes */ + if (uffd_register_memory(uffd_fd, block->host, block->max_length, + UFFDIO_REGISTER_MODE_WP, &uffd_ioctls)) { + goto out; + } + if ((uffd_ioctls & uffd_ioctls_mask) != uffd_ioctls_mask) { + goto out; + } + } + ret = true; + +out: + uffd_close_fd(uffd_fd); + return ret; +} + +static inline void populate_read_range(RAMBlock *block, ram_addr_t offset, + ram_addr_t size) +{ + /* + * We read one byte of each page; this will preallocate page tables if + * required and populate the shared zeropage on MAP_PRIVATE anonymous memory + * where no page was populated yet. This might require adaption when + * supporting other mappings, like shmem. + */ + for (; offset < size; offset += block->page_size) { + char tmp = *((char *)block->host + offset); + + /* Don't optimize the read out */ + asm volatile("" : "+r" (tmp)); + } +} + +static inline int populate_read_section(MemoryRegionSection *section, + void *opaque) +{ + const hwaddr size = int128_get64(section->size); + hwaddr offset = section->offset_within_region; + RAMBlock *block = section->mr->ram_block; + + populate_read_range(block, offset, size); + return 0; +} + +/* + * ram_block_populate_read: preallocate page tables and populate pages in the + * RAM block by reading a byte of each page. + * + * Since it's solely used for userfault_fd WP feature, here we just + * hardcode page size to qemu_real_host_page_size. + * + * @block: RAM block to populate + */ +static void ram_block_populate_read(RAMBlock *rb) +{ + /* + * Skip populating all pages that fall into a discarded range as managed by + * a RamDiscardManager responsible for the mapped memory region of the + * RAMBlock. Such discarded ("logically unplugged") parts of a RAMBlock + * must not get populated automatically. We don't have to track + * modifications via userfaultfd WP reliably, because these pages will + * not be part of the migration stream either way -- see + * ramblock_dirty_bitmap_exclude_discarded_pages(). + * + * Note: The result is only stable while migrating (precopy/postcopy). + */ + if (rb->mr && memory_region_has_ram_discard_manager(rb->mr)) { + RamDiscardManager *rdm = memory_region_get_ram_discard_manager(rb->mr); + MemoryRegionSection section = { + .mr = rb->mr, + .offset_within_region = 0, + .size = rb->mr->size, + }; + + ram_discard_manager_replay_populated(rdm, §ion, + populate_read_section, NULL); + } else { + populate_read_range(rb, 0, rb->used_length); + } +} + +/* + * ram_write_tracking_prepare: prepare for UFFD-WP memory tracking + */ +void ram_write_tracking_prepare(void) +{ + RAMBlock *block; + + RCU_READ_LOCK_GUARD(); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + /* Nothing to do with read-only and MMIO-writable regions */ + if (block->mr->readonly || block->mr->rom_device) { + continue; + } + + /* + * Populate pages of the RAM block before enabling userfault_fd + * write protection. + * + * This stage is required since ioctl(UFFDIO_WRITEPROTECT) with + * UFFDIO_WRITEPROTECT_MODE_WP mode setting would silently skip + * pages with pte_none() entries in page table. + */ + ram_block_populate_read(block); + } +} + +/* + * ram_write_tracking_start: start UFFD-WP memory tracking + * + * Returns 0 for success or negative value in case of error + */ +int ram_write_tracking_start(void) +{ + int uffd_fd; + RAMState *rs = ram_state; + RAMBlock *block; + + /* Open UFFD file descriptor */ + uffd_fd = uffd_create_fd(UFFD_FEATURE_PAGEFAULT_FLAG_WP, true); + if (uffd_fd < 0) { + return uffd_fd; + } + rs->uffdio_fd = uffd_fd; + + RCU_READ_LOCK_GUARD(); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + /* Nothing to do with read-only and MMIO-writable regions */ + if (block->mr->readonly || block->mr->rom_device) { + continue; + } + + /* Register block memory with UFFD to track writes */ + if (uffd_register_memory(rs->uffdio_fd, block->host, + block->max_length, UFFDIO_REGISTER_MODE_WP, NULL)) { + goto fail; + } + /* Apply UFFD write protection to the block memory range */ + if (uffd_change_protection(rs->uffdio_fd, block->host, + block->max_length, true, false)) { + goto fail; + } + block->flags |= RAM_UF_WRITEPROTECT; + memory_region_ref(block->mr); + + trace_ram_write_tracking_ramblock_start(block->idstr, block->page_size, + block->host, block->max_length); + } + + return 0; + +fail: + error_report("ram_write_tracking_start() failed: restoring initial memory state"); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + if ((block->flags & RAM_UF_WRITEPROTECT) == 0) { + continue; + } + /* + * In case some memory block failed to be write-protected + * remove protection and unregister all succeeded RAM blocks + */ + uffd_change_protection(rs->uffdio_fd, block->host, block->max_length, + false, false); + uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length); + /* Cleanup flags and remove reference */ + block->flags &= ~RAM_UF_WRITEPROTECT; + memory_region_unref(block->mr); + } + + uffd_close_fd(uffd_fd); + rs->uffdio_fd = -1; + return -1; +} + +/** + * ram_write_tracking_stop: stop UFFD-WP memory tracking and remove protection + */ +void ram_write_tracking_stop(void) +{ + RAMState *rs = ram_state; + RAMBlock *block; + + RCU_READ_LOCK_GUARD(); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + if ((block->flags & RAM_UF_WRITEPROTECT) == 0) { + continue; + } + /* Remove protection and unregister all affected RAM blocks */ + uffd_change_protection(rs->uffdio_fd, block->host, block->max_length, + false, false); + uffd_unregister_memory(rs->uffdio_fd, block->host, block->max_length); + + trace_ram_write_tracking_ramblock_stop(block->idstr, block->page_size, + block->host, block->max_length); + + /* Cleanup flags and remove reference */ + block->flags &= ~RAM_UF_WRITEPROTECT; + memory_region_unref(block->mr); + } + + /* Finally close UFFD file descriptor */ + uffd_close_fd(rs->uffdio_fd); + rs->uffdio_fd = -1; +} + +#else +/* No target OS support, stubs just fail or ignore */ + +static RAMBlock *poll_fault_page(RAMState *rs, ram_addr_t *offset) +{ + (void) rs; + (void) offset; + + return NULL; +} + +static int ram_save_release_protection(RAMState *rs, PageSearchStatus *pss, + unsigned long start_page) +{ + (void) rs; + (void) pss; + (void) start_page; + + return 0; +} + +bool ram_write_tracking_available(void) +{ + return false; +} + +bool ram_write_tracking_compatible(void) +{ + assert(0); + return false; +} + +int ram_write_tracking_start(void) +{ + assert(0); + return -1; +} + +void ram_write_tracking_stop(void) +{ + assert(0); +} +#endif /* defined(__linux__) */ + +/** + * get_queued_page: unqueue a page from the postcopy requests + * + * Skips pages that are already sent (!dirty) + * + * Returns true if a queued page is found + * + * @rs: current RAM state + * @pss: data about the state of the current dirty page scan + */ +static bool get_queued_page(RAMState *rs, PageSearchStatus *pss) +{ + RAMBlock *block; + ram_addr_t offset; + bool dirty; + + do { + block = unqueue_page(rs, &offset); + /* + * We're sending this page, and since it's postcopy nothing else + * will dirty it, and we must make sure it doesn't get sent again + * even if this queue request was received after the background + * search already sent it. + */ + if (block) { + unsigned long page; + + page = offset >> TARGET_PAGE_BITS; + dirty = test_bit(page, block->bmap); + if (!dirty) { + trace_get_queued_page_not_dirty(block->idstr, (uint64_t)offset, + page); + } else { + trace_get_queued_page(block->idstr, (uint64_t)offset, page); + } + } + + } while (block && !dirty); + + if (!block) { + /* + * Poll write faults too if background snapshot is enabled; that's + * when we have vcpus got blocked by the write protected pages. + */ + block = poll_fault_page(rs, &offset); + } + + if (block) { + /* + * We want the background search to continue from the queued page + * since the guest is likely to want other pages near to the page + * it just requested. + */ + pss->block = block; + pss->page = offset >> TARGET_PAGE_BITS; + + /* + * This unqueued page would break the "one round" check, even is + * really rare. + */ + pss->complete_round = false; + } + + return !!block; +} + +/** + * migration_page_queue_free: drop any remaining pages in the ram + * request queue + * + * It should be empty at the end anyway, but in error cases there may + * be some left. in case that there is any page left, we drop it. + * + */ +static void migration_page_queue_free(RAMState *rs) +{ + struct RAMSrcPageRequest *mspr, *next_mspr; + /* This queue generally should be empty - but in the case of a failed + * migration might have some droppings in. + */ + RCU_READ_LOCK_GUARD(); + QSIMPLEQ_FOREACH_SAFE(mspr, &rs->src_page_requests, next_req, next_mspr) { + memory_region_unref(mspr->rb->mr); + QSIMPLEQ_REMOVE_HEAD(&rs->src_page_requests, next_req); + g_free(mspr); + } +} + +/** + * ram_save_queue_pages: queue the page for transmission + * + * A request from postcopy destination for example. + * + * Returns zero on success or negative on error + * + * @rbname: Name of the RAMBLock of the request. NULL means the + * same that last one. + * @start: starting address from the start of the RAMBlock + * @len: length (in bytes) to send + */ +int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len) +{ + RAMBlock *ramblock; + RAMState *rs = ram_state; + + ram_counters.postcopy_requests++; + RCU_READ_LOCK_GUARD(); + + if (!rbname) { + /* Reuse last RAMBlock */ + ramblock = rs->last_req_rb; + + if (!ramblock) { + /* + * Shouldn't happen, we can't reuse the last RAMBlock if + * it's the 1st request. + */ + error_report("ram_save_queue_pages no previous block"); + return -1; + } + } else { + ramblock = qemu_ram_block_by_name(rbname); + + if (!ramblock) { + /* We shouldn't be asked for a non-existent RAMBlock */ + error_report("ram_save_queue_pages no block '%s'", rbname); + return -1; + } + rs->last_req_rb = ramblock; + } + trace_ram_save_queue_pages(ramblock->idstr, start, len); + if (!offset_in_ramblock(ramblock, start + len - 1)) { + error_report("%s request overrun start=" RAM_ADDR_FMT " len=" + RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT, + __func__, start, len, ramblock->used_length); + return -1; + } + + struct RAMSrcPageRequest *new_entry = + g_malloc0(sizeof(struct RAMSrcPageRequest)); + new_entry->rb = ramblock; + new_entry->offset = start; + new_entry->len = len; + + memory_region_ref(ramblock->mr); + qemu_mutex_lock(&rs->src_page_req_mutex); + QSIMPLEQ_INSERT_TAIL(&rs->src_page_requests, new_entry, next_req); + migration_make_urgent_request(); + qemu_mutex_unlock(&rs->src_page_req_mutex); + + return 0; +} + +static bool save_page_use_compression(RAMState *rs) +{ + if (!migrate_use_compression()) { + return false; + } + + /* + * If xbzrle is enabled (e.g., after first round of migration), stop + * using the data compression. In theory, xbzrle can do better than + * compression. + */ + if (rs->xbzrle_enabled) { + return false; + } + + return true; +} + +/* + * try to compress the page before posting it out, return true if the page + * has been properly handled by compression, otherwise needs other + * paths to handle it + */ +static bool save_compress_page(RAMState *rs, RAMBlock *block, ram_addr_t offset) +{ + if (!save_page_use_compression(rs)) { + return false; + } + + /* + * When starting the process of a new block, the first page of + * the block should be sent out before other pages in the same + * block, and all the pages in last block should have been sent + * out, keeping this order is important, because the 'cont' flag + * is used to avoid resending the block name. + * + * We post the fist page as normal page as compression will take + * much CPU resource. + */ + if (block != rs->last_sent_block) { + flush_compressed_data(rs); + return false; + } + + if (compress_page_with_multi_thread(rs, block, offset) > 0) { + return true; + } + + compression_counters.busy++; + return false; +} + +/** + * ram_save_target_page: save one target page + * + * Returns the number of pages written + * + * @rs: current RAM state + * @pss: data about the page we want to send + * @last_stage: if we are at the completion stage + */ +static int ram_save_target_page(RAMState *rs, PageSearchStatus *pss, + bool last_stage) +{ + RAMBlock *block = pss->block; + ram_addr_t offset = ((ram_addr_t)pss->page) << TARGET_PAGE_BITS; + int res; + + if (control_save_page(rs, block, offset, &res)) { + return res; + } + + if (save_compress_page(rs, block, offset)) { + return 1; + } + + res = save_zero_page(rs, block, offset); + if (res > 0) { + /* Must let xbzrle know, otherwise a previous (now 0'd) cached + * page would be stale + */ + if (!save_page_use_compression(rs)) { + XBZRLE_cache_lock(); + xbzrle_cache_zero_page(rs, block->offset + offset); + XBZRLE_cache_unlock(); + } + ram_release_pages(block->idstr, offset, res); + return res; + } + + /* + * Do not use multifd for: + * 1. Compression as the first page in the new block should be posted out + * before sending the compressed page + * 2. In postcopy as one whole host page should be placed + */ + if (!save_page_use_compression(rs) && migrate_use_multifd() + && !migration_in_postcopy()) { + return ram_save_multifd_page(rs, block, offset); + } + + return ram_save_page(rs, pss, last_stage); +} + +/** + * ram_save_host_page: save a whole host page + * + * Starting at *offset send pages up to the end of the current host + * page. It's valid for the initial offset to point into the middle of + * a host page in which case the remainder of the hostpage is sent. + * Only dirty target pages are sent. Note that the host page size may + * be a huge page for this block. + * The saving stops at the boundary of the used_length of the block + * if the RAMBlock isn't a multiple of the host page size. + * + * Returns the number of pages written or negative on error + * + * @rs: current RAM state + * @ms: current migration state + * @pss: data about the page we want to send + * @last_stage: if we are at the completion stage + */ +static int ram_save_host_page(RAMState *rs, PageSearchStatus *pss, + bool last_stage) +{ + int tmppages, pages = 0; + size_t pagesize_bits = + qemu_ram_pagesize(pss->block) >> TARGET_PAGE_BITS; + unsigned long hostpage_boundary = + QEMU_ALIGN_UP(pss->page + 1, pagesize_bits); + unsigned long start_page = pss->page; + int res; + + if (ramblock_is_ignored(pss->block)) { + error_report("block %s should not be migrated !", pss->block->idstr); + return 0; + } + + do { + /* Check the pages is dirty and if it is send it */ + if (migration_bitmap_clear_dirty(rs, pss->block, pss->page)) { + tmppages = ram_save_target_page(rs, pss, last_stage); + if (tmppages < 0) { + return tmppages; + } + + pages += tmppages; + /* + * Allow rate limiting to happen in the middle of huge pages if + * something is sent in the current iteration. + */ + if (pagesize_bits > 1 && tmppages > 0) { + migration_rate_limit(); + } + } + pss->page = migration_bitmap_find_dirty(rs, pss->block, pss->page); + } while ((pss->page < hostpage_boundary) && + offset_in_ramblock(pss->block, + ((ram_addr_t)pss->page) << TARGET_PAGE_BITS)); + /* The offset we leave with is the min boundary of host page and block */ + pss->page = MIN(pss->page, hostpage_boundary) - 1; + + res = ram_save_release_protection(rs, pss, start_page); + return (res < 0 ? res : pages); +} + +/** + * ram_find_and_save_block: finds a dirty page and sends it to f + * + * Called within an RCU critical section. + * + * Returns the number of pages written where zero means no dirty pages, + * or negative on error + * + * @rs: current RAM state + * @last_stage: if we are at the completion stage + * + * On systems where host-page-size > target-page-size it will send all the + * pages in a host page that are dirty. + */ + +static int ram_find_and_save_block(RAMState *rs, bool last_stage) +{ + PageSearchStatus pss; + int pages = 0; + bool again, found; + + /* No dirty page as there is zero RAM */ + if (!ram_bytes_total()) { + return pages; + } + + pss.block = rs->last_seen_block; + pss.page = rs->last_page; + pss.complete_round = false; + + if (!pss.block) { + pss.block = QLIST_FIRST_RCU(&ram_list.blocks); + } + + do { + again = true; + found = get_queued_page(rs, &pss); + + if (!found) { + /* priority queue empty, so just search for something dirty */ + found = find_dirty_block(rs, &pss, &again); + } + + if (found) { + pages = ram_save_host_page(rs, &pss, last_stage); + } + } while (!pages && again); + + rs->last_seen_block = pss.block; + rs->last_page = pss.page; + + return pages; +} + +void acct_update_position(QEMUFile *f, size_t size, bool zero) +{ + uint64_t pages = size / TARGET_PAGE_SIZE; + + if (zero) { + ram_counters.duplicate += pages; + } else { + ram_counters.normal += pages; + ram_counters.transferred += size; + qemu_update_position(f, size); + } +} + +static uint64_t ram_bytes_total_common(bool count_ignored) +{ + RAMBlock *block; + uint64_t total = 0; + + RCU_READ_LOCK_GUARD(); + + if (count_ignored) { + RAMBLOCK_FOREACH_MIGRATABLE(block) { + total += block->used_length; + } + } else { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + total += block->used_length; + } + } + return total; +} + +uint64_t ram_bytes_total(void) +{ + return ram_bytes_total_common(false); +} + +static void xbzrle_load_setup(void) +{ + XBZRLE.decoded_buf = g_malloc(TARGET_PAGE_SIZE); +} + +static void xbzrle_load_cleanup(void) +{ + g_free(XBZRLE.decoded_buf); + XBZRLE.decoded_buf = NULL; +} + +static void ram_state_cleanup(RAMState **rsp) +{ + if (*rsp) { + migration_page_queue_free(*rsp); + qemu_mutex_destroy(&(*rsp)->bitmap_mutex); + qemu_mutex_destroy(&(*rsp)->src_page_req_mutex); + g_free(*rsp); + *rsp = NULL; + } +} + +static void xbzrle_cleanup(void) +{ + XBZRLE_cache_lock(); + if (XBZRLE.cache) { + cache_fini(XBZRLE.cache); + g_free(XBZRLE.encoded_buf); + g_free(XBZRLE.current_buf); + g_free(XBZRLE.zero_target_page); + XBZRLE.cache = NULL; + XBZRLE.encoded_buf = NULL; + XBZRLE.current_buf = NULL; + XBZRLE.zero_target_page = NULL; + } + XBZRLE_cache_unlock(); +} + +static void ram_save_cleanup(void *opaque) +{ + RAMState **rsp = opaque; + RAMBlock *block; + + /* We don't use dirty log with background snapshots */ + if (!migrate_background_snapshot()) { + /* caller have hold iothread lock or is in a bh, so there is + * no writing race against the migration bitmap + */ + if (global_dirty_tracking & GLOBAL_DIRTY_MIGRATION) { + /* + * do not stop dirty log without starting it, since + * memory_global_dirty_log_stop will assert that + * memory_global_dirty_log_start/stop used in pairs + */ + memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION); + } + } + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + g_free(block->clear_bmap); + block->clear_bmap = NULL; + g_free(block->bmap); + block->bmap = NULL; + } + + xbzrle_cleanup(); + compress_threads_save_cleanup(); + ram_state_cleanup(rsp); +} + +static void ram_state_reset(RAMState *rs) +{ + rs->last_seen_block = NULL; + rs->last_sent_block = NULL; + rs->last_page = 0; + rs->last_version = ram_list.version; + rs->xbzrle_enabled = false; +} + +#define MAX_WAIT 50 /* ms, half buffered_file limit */ + +/* + * 'expected' is the value you expect the bitmap mostly to be full + * of; it won't bother printing lines that are all this value. + * If 'todump' is null the migration bitmap is dumped. + */ +void ram_debug_dump_bitmap(unsigned long *todump, bool expected, + unsigned long pages) +{ + int64_t cur; + int64_t linelen = 128; + char linebuf[129]; + + for (cur = 0; cur < pages; cur += linelen) { + int64_t curb; + bool found = false; + /* + * Last line; catch the case where the line length + * is longer than remaining ram + */ + if (cur + linelen > pages) { + linelen = pages - cur; + } + for (curb = 0; curb < linelen; curb++) { + bool thisbit = test_bit(cur + curb, todump); + linebuf[curb] = thisbit ? '1' : '.'; + found = found || (thisbit != expected); + } + if (found) { + linebuf[curb] = '\0'; + fprintf(stderr, "0x%08" PRIx64 " : %s\n", cur, linebuf); + } + } +} + +/* **** functions for postcopy ***** */ + +void ram_postcopy_migrated_memory_release(MigrationState *ms) +{ + struct RAMBlock *block; + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + unsigned long *bitmap = block->bmap; + unsigned long range = block->used_length >> TARGET_PAGE_BITS; + unsigned long run_start = find_next_zero_bit(bitmap, range, 0); + + while (run_start < range) { + unsigned long run_end = find_next_bit(bitmap, range, run_start + 1); + ram_discard_range(block->idstr, + ((ram_addr_t)run_start) << TARGET_PAGE_BITS, + ((ram_addr_t)(run_end - run_start)) + << TARGET_PAGE_BITS); + run_start = find_next_zero_bit(bitmap, range, run_end + 1); + } + } +} + +/** + * postcopy_send_discard_bm_ram: discard a RAMBlock + * + * Returns zero on success + * + * Callback from postcopy_each_ram_send_discard for each RAMBlock + * + * @ms: current migration state + * @block: RAMBlock to discard + */ +static int postcopy_send_discard_bm_ram(MigrationState *ms, RAMBlock *block) +{ + unsigned long end = block->used_length >> TARGET_PAGE_BITS; + unsigned long current; + unsigned long *bitmap = block->bmap; + + for (current = 0; current < end; ) { + unsigned long one = find_next_bit(bitmap, end, current); + unsigned long zero, discard_length; + + if (one >= end) { + break; + } + + zero = find_next_zero_bit(bitmap, end, one + 1); + + if (zero >= end) { + discard_length = end - one; + } else { + discard_length = zero - one; + } + postcopy_discard_send_range(ms, one, discard_length); + current = one + discard_length; + } + + return 0; +} + +/** + * postcopy_each_ram_send_discard: discard all RAMBlocks + * + * Returns 0 for success or negative for error + * + * Utility for the outgoing postcopy code. + * Calls postcopy_send_discard_bm_ram for each RAMBlock + * passing it bitmap indexes and name. + * (qemu_ram_foreach_block ends up passing unscaled lengths + * which would mean postcopy code would have to deal with target page) + * + * @ms: current migration state + */ +static int postcopy_each_ram_send_discard(MigrationState *ms) +{ + struct RAMBlock *block; + int ret; + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + postcopy_discard_send_init(ms, block->idstr); + + /* + * Postcopy sends chunks of bitmap over the wire, but it + * just needs indexes at this point, avoids it having + * target page specific code. + */ + ret = postcopy_send_discard_bm_ram(ms, block); + postcopy_discard_send_finish(ms); + if (ret) { + return ret; + } + } + + return 0; +} + +/** + * postcopy_chunk_hostpages_pass: canonicalize bitmap in hostpages + * + * Helper for postcopy_chunk_hostpages; it's called twice to + * canonicalize the two bitmaps, that are similar, but one is + * inverted. + * + * Postcopy requires that all target pages in a hostpage are dirty or + * clean, not a mix. This function canonicalizes the bitmaps. + * + * @ms: current migration state + * @block: block that contains the page we want to canonicalize + */ +static void postcopy_chunk_hostpages_pass(MigrationState *ms, RAMBlock *block) +{ + RAMState *rs = ram_state; + unsigned long *bitmap = block->bmap; + unsigned int host_ratio = block->page_size / TARGET_PAGE_SIZE; + unsigned long pages = block->used_length >> TARGET_PAGE_BITS; + unsigned long run_start; + + if (block->page_size == TARGET_PAGE_SIZE) { + /* Easy case - TPS==HPS for a non-huge page RAMBlock */ + return; + } + + /* Find a dirty page */ + run_start = find_next_bit(bitmap, pages, 0); + + while (run_start < pages) { + + /* + * If the start of this run of pages is in the middle of a host + * page, then we need to fixup this host page. + */ + if (QEMU_IS_ALIGNED(run_start, host_ratio)) { + /* Find the end of this run */ + run_start = find_next_zero_bit(bitmap, pages, run_start + 1); + /* + * If the end isn't at the start of a host page, then the + * run doesn't finish at the end of a host page + * and we need to discard. + */ + } + + if (!QEMU_IS_ALIGNED(run_start, host_ratio)) { + unsigned long page; + unsigned long fixup_start_addr = QEMU_ALIGN_DOWN(run_start, + host_ratio); + run_start = QEMU_ALIGN_UP(run_start, host_ratio); + + /* Clean up the bitmap */ + for (page = fixup_start_addr; + page < fixup_start_addr + host_ratio; page++) { + /* + * Remark them as dirty, updating the count for any pages + * that weren't previously dirty. + */ + rs->migration_dirty_pages += !test_and_set_bit(page, bitmap); + } + } + + /* Find the next dirty page for the next iteration */ + run_start = find_next_bit(bitmap, pages, run_start); + } +} + +/** + * postcopy_chunk_hostpages: discard any partially sent host page + * + * Utility for the outgoing postcopy code. + * + * Discard any partially sent host-page size chunks, mark any partially + * dirty host-page size chunks as all dirty. In this case the host-page + * is the host-page for the particular RAMBlock, i.e. it might be a huge page + * + * Returns zero on success + * + * @ms: current migration state + * @block: block we want to work with + */ +static int postcopy_chunk_hostpages(MigrationState *ms, RAMBlock *block) +{ + postcopy_discard_send_init(ms, block->idstr); + + /* + * Ensure that all partially dirty host pages are made fully dirty. + */ + postcopy_chunk_hostpages_pass(ms, block); + + postcopy_discard_send_finish(ms); + return 0; +} + +/** + * ram_postcopy_send_discard_bitmap: transmit the discard bitmap + * + * Returns zero on success + * + * Transmit the set of pages to be discarded after precopy to the target + * these are pages that: + * a) Have been previously transmitted but are now dirty again + * b) Pages that have never been transmitted, this ensures that + * any pages on the destination that have been mapped by background + * tasks get discarded (transparent huge pages is the specific concern) + * Hopefully this is pretty sparse + * + * @ms: current migration state + */ +int ram_postcopy_send_discard_bitmap(MigrationState *ms) +{ + RAMState *rs = ram_state; + RAMBlock *block; + int ret; + + RCU_READ_LOCK_GUARD(); + + /* This should be our last sync, the src is now paused */ + migration_bitmap_sync(rs); + + /* Easiest way to make sure we don't resume in the middle of a host-page */ + rs->last_seen_block = NULL; + rs->last_sent_block = NULL; + rs->last_page = 0; + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + /* Deal with TPS != HPS and huge pages */ + ret = postcopy_chunk_hostpages(ms, block); + if (ret) { + return ret; + } + +#ifdef DEBUG_POSTCOPY + ram_debug_dump_bitmap(block->bmap, true, + block->used_length >> TARGET_PAGE_BITS); +#endif + } + trace_ram_postcopy_send_discard_bitmap(); + + return postcopy_each_ram_send_discard(ms); +} + +/** + * ram_discard_range: discard dirtied pages at the beginning of postcopy + * + * Returns zero on success + * + * @rbname: name of the RAMBlock of the request. NULL means the + * same that last one. + * @start: RAMBlock starting page + * @length: RAMBlock size + */ +int ram_discard_range(const char *rbname, uint64_t start, size_t length) +{ + trace_ram_discard_range(rbname, start, length); + + RCU_READ_LOCK_GUARD(); + RAMBlock *rb = qemu_ram_block_by_name(rbname); + + if (!rb) { + error_report("ram_discard_range: Failed to find block '%s'", rbname); + return -1; + } + + /* + * On source VM, we don't need to update the received bitmap since + * we don't even have one. + */ + if (rb->receivedmap) { + bitmap_clear(rb->receivedmap, start >> qemu_target_page_bits(), + length >> qemu_target_page_bits()); + } + + return ram_block_discard_range(rb, start, length); +} + +/* + * For every allocation, we will try not to crash the VM if the + * allocation failed. + */ +static int xbzrle_init(void) +{ + Error *local_err = NULL; + + if (!migrate_use_xbzrle()) { + return 0; + } + + XBZRLE_cache_lock(); + + XBZRLE.zero_target_page = g_try_malloc0(TARGET_PAGE_SIZE); + if (!XBZRLE.zero_target_page) { + error_report("%s: Error allocating zero page", __func__); + goto err_out; + } + + XBZRLE.cache = cache_init(migrate_xbzrle_cache_size(), + TARGET_PAGE_SIZE, &local_err); + if (!XBZRLE.cache) { + error_report_err(local_err); + goto free_zero_page; + } + + XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE); + if (!XBZRLE.encoded_buf) { + error_report("%s: Error allocating encoded_buf", __func__); + goto free_cache; + } + + XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE); + if (!XBZRLE.current_buf) { + error_report("%s: Error allocating current_buf", __func__); + goto free_encoded_buf; + } + + /* We are all good */ + XBZRLE_cache_unlock(); + return 0; + +free_encoded_buf: + g_free(XBZRLE.encoded_buf); + XBZRLE.encoded_buf = NULL; +free_cache: + cache_fini(XBZRLE.cache); + XBZRLE.cache = NULL; +free_zero_page: + g_free(XBZRLE.zero_target_page); + XBZRLE.zero_target_page = NULL; +err_out: + XBZRLE_cache_unlock(); + return -ENOMEM; +} + +static int ram_state_init(RAMState **rsp) +{ + *rsp = g_try_new0(RAMState, 1); + + if (!*rsp) { + error_report("%s: Init ramstate fail", __func__); + return -1; + } + + qemu_mutex_init(&(*rsp)->bitmap_mutex); + qemu_mutex_init(&(*rsp)->src_page_req_mutex); + QSIMPLEQ_INIT(&(*rsp)->src_page_requests); + + /* + * Count the total number of pages used by ram blocks not including any + * gaps due to alignment or unplugs. + * This must match with the initial values of dirty bitmap. + */ + (*rsp)->migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS; + ram_state_reset(*rsp); + + return 0; +} + +static void ram_list_init_bitmaps(void) +{ + MigrationState *ms = migrate_get_current(); + RAMBlock *block; + unsigned long pages; + uint8_t shift; + + /* Skip setting bitmap if there is no RAM */ + if (ram_bytes_total()) { + shift = ms->clear_bitmap_shift; + if (shift > CLEAR_BITMAP_SHIFT_MAX) { + error_report("clear_bitmap_shift (%u) too big, using " + "max value (%u)", shift, CLEAR_BITMAP_SHIFT_MAX); + shift = CLEAR_BITMAP_SHIFT_MAX; + } else if (shift < CLEAR_BITMAP_SHIFT_MIN) { + error_report("clear_bitmap_shift (%u) too small, using " + "min value (%u)", shift, CLEAR_BITMAP_SHIFT_MIN); + shift = CLEAR_BITMAP_SHIFT_MIN; + } + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + pages = block->max_length >> TARGET_PAGE_BITS; + /* + * The initial dirty bitmap for migration must be set with all + * ones to make sure we'll migrate every guest RAM page to + * destination. + * Here we set RAMBlock.bmap all to 1 because when rebegin a + * new migration after a failed migration, ram_list. + * dirty_memory[DIRTY_MEMORY_MIGRATION] don't include the whole + * guest memory. + */ + block->bmap = bitmap_new(pages); + bitmap_set(block->bmap, 0, pages); + block->clear_bmap_shift = shift; + block->clear_bmap = bitmap_new(clear_bmap_size(pages, shift)); + } + } +} + +static void migration_bitmap_clear_discarded_pages(RAMState *rs) +{ + unsigned long pages; + RAMBlock *rb; + + RCU_READ_LOCK_GUARD(); + + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { + pages = ramblock_dirty_bitmap_clear_discarded_pages(rb); + rs->migration_dirty_pages -= pages; + } +} + +static void ram_init_bitmaps(RAMState *rs) +{ + /* For memory_global_dirty_log_start below. */ + qemu_mutex_lock_iothread(); + qemu_mutex_lock_ramlist(); + + WITH_RCU_READ_LOCK_GUARD() { + ram_list_init_bitmaps(); + /* We don't use dirty log with background snapshots */ + if (!migrate_background_snapshot()) { + memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION); + migration_bitmap_sync_precopy(rs); + } + } + qemu_mutex_unlock_ramlist(); + qemu_mutex_unlock_iothread(); + + /* + * After an eventual first bitmap sync, fixup the initial bitmap + * containing all 1s to exclude any discarded pages from migration. + */ + migration_bitmap_clear_discarded_pages(rs); +} + +static int ram_init_all(RAMState **rsp) +{ + if (ram_state_init(rsp)) { + return -1; + } + + if (xbzrle_init()) { + ram_state_cleanup(rsp); + return -1; + } + + ram_init_bitmaps(*rsp); + + return 0; +} + +static void ram_state_resume_prepare(RAMState *rs, QEMUFile *out) +{ + RAMBlock *block; + uint64_t pages = 0; + + /* + * Postcopy is not using xbzrle/compression, so no need for that. + * Also, since source are already halted, we don't need to care + * about dirty page logging as well. + */ + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + pages += bitmap_count_one(block->bmap, + block->used_length >> TARGET_PAGE_BITS); + } + + /* This may not be aligned with current bitmaps. Recalculate. */ + rs->migration_dirty_pages = pages; + + ram_state_reset(rs); + + /* Update RAMState cache of output QEMUFile */ + rs->f = out; + + trace_ram_state_resume_prepare(pages); +} + +/* + * This function clears bits of the free pages reported by the caller from the + * migration dirty bitmap. @addr is the host address corresponding to the + * start of the continuous guest free pages, and @len is the total bytes of + * those pages. + */ +void qemu_guest_free_page_hint(void *addr, size_t len) +{ + RAMBlock *block; + ram_addr_t offset; + size_t used_len, start, npages; + MigrationState *s = migrate_get_current(); + + /* This function is currently expected to be used during live migration */ + if (!migration_is_setup_or_active(s->state)) { + return; + } + + for (; len > 0; len -= used_len, addr += used_len) { + block = qemu_ram_block_from_host(addr, false, &offset); + if (unlikely(!block || offset >= block->used_length)) { + /* + * The implementation might not support RAMBlock resize during + * live migration, but it could happen in theory with future + * updates. So we add a check here to capture that case. + */ + error_report_once("%s unexpected error", __func__); + return; + } + + if (len <= block->used_length - offset) { + used_len = len; + } else { + used_len = block->used_length - offset; + } + + start = offset >> TARGET_PAGE_BITS; + npages = used_len >> TARGET_PAGE_BITS; + + qemu_mutex_lock(&ram_state->bitmap_mutex); + /* + * The skipped free pages are equavalent to be sent from clear_bmap's + * perspective, so clear the bits from the memory region bitmap which + * are initially set. Otherwise those skipped pages will be sent in + * the next round after syncing from the memory region bitmap. + */ + migration_clear_memory_region_dirty_bitmap_range(block, start, npages); + ram_state->migration_dirty_pages -= + bitmap_count_one_with_offset(block->bmap, start, npages); + bitmap_clear(block->bmap, start, npages); + qemu_mutex_unlock(&ram_state->bitmap_mutex); + } +} + +/* + * Each of ram_save_setup, ram_save_iterate and ram_save_complete has + * long-running RCU critical section. When rcu-reclaims in the code + * start to become numerous it will be necessary to reduce the + * granularity of these critical sections. + */ + +/** + * ram_save_setup: Setup RAM for migration + * + * Returns zero to indicate success and negative for error + * + * @f: QEMUFile where to send the data + * @opaque: RAMState pointer + */ +static int ram_save_setup(QEMUFile *f, void *opaque) +{ + RAMState **rsp = opaque; + RAMBlock *block; + + if (compress_threads_save_setup()) { + return -1; + } + + /* migration has already setup the bitmap, reuse it. */ + if (!migration_in_colo_state()) { + if (ram_init_all(rsp) != 0) { + compress_threads_save_cleanup(); + return -1; + } + } + (*rsp)->f = f; + + WITH_RCU_READ_LOCK_GUARD() { + qemu_put_be64(f, ram_bytes_total_common(true) | RAM_SAVE_FLAG_MEM_SIZE); + + RAMBLOCK_FOREACH_MIGRATABLE(block) { + qemu_put_byte(f, strlen(block->idstr)); + qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr)); + qemu_put_be64(f, block->used_length); + if (migrate_postcopy_ram() && block->page_size != + qemu_host_page_size) { + qemu_put_be64(f, block->page_size); + } + if (migrate_ignore_shared()) { + qemu_put_be64(f, block->mr->addr); + } + } + } + + ram_control_before_iterate(f, RAM_CONTROL_SETUP); + ram_control_after_iterate(f, RAM_CONTROL_SETUP); + + multifd_send_sync_main(f); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + qemu_fflush(f); + + return 0; +} + +/** + * ram_save_iterate: iterative stage for migration + * + * Returns zero to indicate success and negative for error + * + * @f: QEMUFile where to send the data + * @opaque: RAMState pointer + */ +static int ram_save_iterate(QEMUFile *f, void *opaque) +{ + RAMState **temp = opaque; + RAMState *rs = *temp; + int ret = 0; + int i; + int64_t t0; + int done = 0; + + if (blk_mig_bulk_active()) { + /* Avoid transferring ram during bulk phase of block migration as + * the bulk phase will usually take a long time and transferring + * ram updates during that time is pointless. */ + goto out; + } + + /* + * We'll take this lock a little bit long, but it's okay for two reasons. + * Firstly, the only possible other thread to take it is who calls + * qemu_guest_free_page_hint(), which should be rare; secondly, see + * MAX_WAIT (if curious, further see commit 4508bd9ed8053ce) below, which + * guarantees that we'll at least released it in a regular basis. + */ + qemu_mutex_lock(&rs->bitmap_mutex); + WITH_RCU_READ_LOCK_GUARD() { + if (ram_list.version != rs->last_version) { + ram_state_reset(rs); + } + + /* Read version before ram_list.blocks */ + smp_rmb(); + + ram_control_before_iterate(f, RAM_CONTROL_ROUND); + + t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); + i = 0; + while ((ret = qemu_file_rate_limit(f)) == 0 || + !QSIMPLEQ_EMPTY(&rs->src_page_requests)) { + int pages; + + if (qemu_file_get_error(f)) { + break; + } + + pages = ram_find_and_save_block(rs, false); + /* no more pages to sent */ + if (pages == 0) { + done = 1; + break; + } + + if (pages < 0) { + qemu_file_set_error(f, pages); + break; + } + + rs->target_page_count += pages; + + /* + * During postcopy, it is necessary to make sure one whole host + * page is sent in one chunk. + */ + if (migrate_postcopy_ram()) { + flush_compressed_data(rs); + } + + /* + * we want to check in the 1st loop, just in case it was the 1st + * time and we had to sync the dirty bitmap. + * qemu_clock_get_ns() is a bit expensive, so we only check each + * some iterations + */ + if ((i & 63) == 0) { + uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / + 1000000; + if (t1 > MAX_WAIT) { + trace_ram_save_iterate_big_wait(t1, i); + break; + } + } + i++; + } + } + qemu_mutex_unlock(&rs->bitmap_mutex); + + /* + * Must occur before EOS (or any QEMUFile operation) + * because of RDMA protocol. + */ + ram_control_after_iterate(f, RAM_CONTROL_ROUND); + +out: + if (ret >= 0 + && migration_is_setup_or_active(migrate_get_current()->state)) { + multifd_send_sync_main(rs->f); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + qemu_fflush(f); + ram_counters.transferred += 8; + + ret = qemu_file_get_error(f); + } + if (ret < 0) { + return ret; + } + + return done; +} + +/** + * ram_save_complete: function called to send the remaining amount of ram + * + * Returns zero to indicate success or negative on error + * + * Called with iothread lock + * + * @f: QEMUFile where to send the data + * @opaque: RAMState pointer + */ +static int ram_save_complete(QEMUFile *f, void *opaque) +{ + RAMState **temp = opaque; + RAMState *rs = *temp; + int ret = 0; + + WITH_RCU_READ_LOCK_GUARD() { + if (!migration_in_postcopy()) { + migration_bitmap_sync_precopy(rs); + } + + ram_control_before_iterate(f, RAM_CONTROL_FINISH); + + /* try transferring iterative blocks of memory */ + + /* flush all remaining blocks regardless of rate limiting */ + while (true) { + int pages; + + pages = ram_find_and_save_block(rs, !migration_in_colo_state()); + /* no more blocks to sent */ + if (pages == 0) { + break; + } + if (pages < 0) { + ret = pages; + break; + } + } + + flush_compressed_data(rs); + ram_control_after_iterate(f, RAM_CONTROL_FINISH); + } + + if (ret >= 0) { + multifd_send_sync_main(rs->f); + qemu_put_be64(f, RAM_SAVE_FLAG_EOS); + qemu_fflush(f); + } + + return ret; +} + +static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size, + uint64_t *res_precopy_only, + uint64_t *res_compatible, + uint64_t *res_postcopy_only) +{ + RAMState **temp = opaque; + RAMState *rs = *temp; + uint64_t remaining_size; + + remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; + + if (!migration_in_postcopy() && + remaining_size < max_size) { + qemu_mutex_lock_iothread(); + WITH_RCU_READ_LOCK_GUARD() { + migration_bitmap_sync_precopy(rs); + } + qemu_mutex_unlock_iothread(); + remaining_size = rs->migration_dirty_pages * TARGET_PAGE_SIZE; + } + + if (migrate_postcopy_ram()) { + /* We can do postcopy, and all the data is postcopiable */ + *res_compatible += remaining_size; + } else { + *res_precopy_only += remaining_size; + } +} + +static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host) +{ + unsigned int xh_len; + int xh_flags; + uint8_t *loaded_data; + + /* extract RLE header */ + xh_flags = qemu_get_byte(f); + xh_len = qemu_get_be16(f); + + if (xh_flags != ENCODING_FLAG_XBZRLE) { + error_report("Failed to load XBZRLE page - wrong compression!"); + return -1; + } + + if (xh_len > TARGET_PAGE_SIZE) { + error_report("Failed to load XBZRLE page - len overflow!"); + return -1; + } + loaded_data = XBZRLE.decoded_buf; + /* load data and decode */ + /* it can change loaded_data to point to an internal buffer */ + qemu_get_buffer_in_place(f, &loaded_data, xh_len); + + /* decode RLE */ + if (xbzrle_decode_buffer(loaded_data, xh_len, host, + TARGET_PAGE_SIZE) == -1) { + error_report("Failed to load XBZRLE page - decode error!"); + return -1; + } + + return 0; +} + +/** + * ram_block_from_stream: read a RAMBlock id from the migration stream + * + * Must be called from within a rcu critical section. + * + * Returns a pointer from within the RCU-protected ram_list. + * + * @f: QEMUFile where to read the data from + * @flags: Page flags (mostly to see if it's a continuation of previous block) + */ +static inline RAMBlock *ram_block_from_stream(QEMUFile *f, int flags) +{ + static RAMBlock *block; + char id[256]; + uint8_t len; + + if (flags & RAM_SAVE_FLAG_CONTINUE) { + if (!block) { + error_report("Ack, bad migration stream!"); + return NULL; + } + return block; + } + + len = qemu_get_byte(f); + qemu_get_buffer(f, (uint8_t *)id, len); + id[len] = 0; + + block = qemu_ram_block_by_name(id); + if (!block) { + error_report("Can't find block %s", id); + return NULL; + } + + if (ramblock_is_ignored(block)) { + error_report("block %s should not be migrated !", id); + return NULL; + } + + return block; +} + +static inline void *host_from_ram_block_offset(RAMBlock *block, + ram_addr_t offset) +{ + if (!offset_in_ramblock(block, offset)) { + return NULL; + } + + return block->host + offset; +} + +static void *host_page_from_ram_block_offset(RAMBlock *block, + ram_addr_t offset) +{ + /* Note: Explicitly no check against offset_in_ramblock(). */ + return (void *)QEMU_ALIGN_DOWN((uintptr_t)(block->host + offset), + block->page_size); +} + +static ram_addr_t host_page_offset_from_ram_block_offset(RAMBlock *block, + ram_addr_t offset) +{ + return ((uintptr_t)block->host + offset) & (block->page_size - 1); +} + +static inline void *colo_cache_from_block_offset(RAMBlock *block, + ram_addr_t offset, bool record_bitmap) +{ + if (!offset_in_ramblock(block, offset)) { + return NULL; + } + if (!block->colo_cache) { + error_report("%s: colo_cache is NULL in block :%s", + __func__, block->idstr); + return NULL; + } + + /* + * During colo checkpoint, we need bitmap of these migrated pages. + * It help us to decide which pages in ram cache should be flushed + * into VM's RAM later. + */ + if (record_bitmap && + !test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) { + ram_state->migration_dirty_pages++; + } + return block->colo_cache + offset; +} + +/** + * ram_handle_compressed: handle the zero page case + * + * If a page (or a whole RDMA chunk) has been + * determined to be zero, then zap it. + * + * @host: host address for the zero page + * @ch: what the page is filled from. We only support zero + * @size: size of the zero page + */ +void ram_handle_compressed(void *host, uint8_t ch, uint64_t size) +{ + if (ch != 0 || !is_zero_range(host, size)) { + memset(host, ch, size); + } +} + +/* return the size after decompression, or negative value on error */ +static int +qemu_uncompress_data(z_stream *stream, uint8_t *dest, size_t dest_len, + const uint8_t *source, size_t source_len) +{ + int err; + + err = inflateReset(stream); + if (err != Z_OK) { + return -1; + } + + stream->avail_in = source_len; + stream->next_in = (uint8_t *)source; + stream->avail_out = dest_len; + stream->next_out = dest; + + err = inflate(stream, Z_NO_FLUSH); + if (err != Z_STREAM_END) { + return -1; + } + + return stream->total_out; +} + +static void *do_data_decompress(void *opaque) +{ + DecompressParam *param = opaque; + unsigned long pagesize; + uint8_t *des; + int len, ret; + + qemu_mutex_lock(¶m->mutex); + while (!param->quit) { + if (param->des) { + des = param->des; + len = param->len; + param->des = 0; + qemu_mutex_unlock(¶m->mutex); + + pagesize = TARGET_PAGE_SIZE; + + ret = qemu_uncompress_data(¶m->stream, des, pagesize, + param->compbuf, len); + if (ret < 0 && migrate_get_current()->decompress_error_check) { + error_report("decompress data failed"); + qemu_file_set_error(decomp_file, ret); + } + + qemu_mutex_lock(&decomp_done_lock); + param->done = true; + qemu_cond_signal(&decomp_done_cond); + qemu_mutex_unlock(&decomp_done_lock); + + qemu_mutex_lock(¶m->mutex); + } else { + qemu_cond_wait(¶m->cond, ¶m->mutex); + } + } + qemu_mutex_unlock(¶m->mutex); + + return NULL; +} + +static int wait_for_decompress_done(void) +{ + int idx, thread_count; + + if (!migrate_use_compression()) { + return 0; + } + + thread_count = migrate_decompress_threads(); + qemu_mutex_lock(&decomp_done_lock); + for (idx = 0; idx < thread_count; idx++) { + while (!decomp_param[idx].done) { + qemu_cond_wait(&decomp_done_cond, &decomp_done_lock); + } + } + qemu_mutex_unlock(&decomp_done_lock); + return qemu_file_get_error(decomp_file); +} + +static void compress_threads_load_cleanup(void) +{ + int i, thread_count; + + if (!migrate_use_compression()) { + return; + } + thread_count = migrate_decompress_threads(); + for (i = 0; i < thread_count; i++) { + /* + * we use it as a indicator which shows if the thread is + * properly init'd or not + */ + if (!decomp_param[i].compbuf) { + break; + } + + qemu_mutex_lock(&decomp_param[i].mutex); + decomp_param[i].quit = true; + qemu_cond_signal(&decomp_param[i].cond); + qemu_mutex_unlock(&decomp_param[i].mutex); + } + for (i = 0; i < thread_count; i++) { + if (!decomp_param[i].compbuf) { + break; + } + + qemu_thread_join(decompress_threads + i); + qemu_mutex_destroy(&decomp_param[i].mutex); + qemu_cond_destroy(&decomp_param[i].cond); + inflateEnd(&decomp_param[i].stream); + g_free(decomp_param[i].compbuf); + decomp_param[i].compbuf = NULL; + } + g_free(decompress_threads); + g_free(decomp_param); + decompress_threads = NULL; + decomp_param = NULL; + decomp_file = NULL; +} + +static int compress_threads_load_setup(QEMUFile *f) +{ + int i, thread_count; + + if (!migrate_use_compression()) { + return 0; + } + + thread_count = migrate_decompress_threads(); + decompress_threads = g_new0(QemuThread, thread_count); + decomp_param = g_new0(DecompressParam, thread_count); + qemu_mutex_init(&decomp_done_lock); + qemu_cond_init(&decomp_done_cond); + decomp_file = f; + for (i = 0; i < thread_count; i++) { + if (inflateInit(&decomp_param[i].stream) != Z_OK) { + goto exit; + } + + decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE)); + qemu_mutex_init(&decomp_param[i].mutex); + qemu_cond_init(&decomp_param[i].cond); + decomp_param[i].done = true; + decomp_param[i].quit = false; + qemu_thread_create(decompress_threads + i, "decompress", + do_data_decompress, decomp_param + i, + QEMU_THREAD_JOINABLE); + } + return 0; +exit: + compress_threads_load_cleanup(); + return -1; +} + +static void decompress_data_with_multi_threads(QEMUFile *f, + void *host, int len) +{ + int idx, thread_count; + + thread_count = migrate_decompress_threads(); + QEMU_LOCK_GUARD(&decomp_done_lock); + while (true) { + for (idx = 0; idx < thread_count; idx++) { + if (decomp_param[idx].done) { + decomp_param[idx].done = false; + qemu_mutex_lock(&decomp_param[idx].mutex); + qemu_get_buffer(f, decomp_param[idx].compbuf, len); + decomp_param[idx].des = host; + decomp_param[idx].len = len; + qemu_cond_signal(&decomp_param[idx].cond); + qemu_mutex_unlock(&decomp_param[idx].mutex); + break; + } + } + if (idx < thread_count) { + break; + } else { + qemu_cond_wait(&decomp_done_cond, &decomp_done_lock); + } + } +} + +static void colo_init_ram_state(void) +{ + ram_state_init(&ram_state); +} + +/* + * colo cache: this is for secondary VM, we cache the whole + * memory of the secondary VM, it is need to hold the global lock + * to call this helper. + */ +int colo_init_ram_cache(void) +{ + RAMBlock *block; + + WITH_RCU_READ_LOCK_GUARD() { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + block->colo_cache = qemu_anon_ram_alloc(block->used_length, + NULL, false, false); + if (!block->colo_cache) { + error_report("%s: Can't alloc memory for COLO cache of block %s," + "size 0x" RAM_ADDR_FMT, __func__, block->idstr, + block->used_length); + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + if (block->colo_cache) { + qemu_anon_ram_free(block->colo_cache, block->used_length); + block->colo_cache = NULL; + } + } + return -errno; + } + if (!machine_dump_guest_core(current_machine)) { + qemu_madvise(block->colo_cache, block->used_length, + QEMU_MADV_DONTDUMP); + } + } + } + + /* + * Record the dirty pages that sent by PVM, we use this dirty bitmap together + * with to decide which page in cache should be flushed into SVM's RAM. Here + * we use the same name 'ram_bitmap' as for migration. + */ + if (ram_bytes_total()) { + RAMBlock *block; + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + unsigned long pages = block->max_length >> TARGET_PAGE_BITS; + block->bmap = bitmap_new(pages); + } + } + + colo_init_ram_state(); + return 0; +} + +/* TODO: duplicated with ram_init_bitmaps */ +void colo_incoming_start_dirty_log(void) +{ + RAMBlock *block = NULL; + /* For memory_global_dirty_log_start below. */ + qemu_mutex_lock_iothread(); + qemu_mutex_lock_ramlist(); + + memory_global_dirty_log_sync(); + WITH_RCU_READ_LOCK_GUARD() { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + ramblock_sync_dirty_bitmap(ram_state, block); + /* Discard this dirty bitmap record */ + bitmap_zero(block->bmap, block->max_length >> TARGET_PAGE_BITS); + } + memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION); + } + ram_state->migration_dirty_pages = 0; + qemu_mutex_unlock_ramlist(); + qemu_mutex_unlock_iothread(); +} + +/* It is need to hold the global lock to call this helper */ +void colo_release_ram_cache(void) +{ + RAMBlock *block; + + memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION); + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + g_free(block->bmap); + block->bmap = NULL; + } + + WITH_RCU_READ_LOCK_GUARD() { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + if (block->colo_cache) { + qemu_anon_ram_free(block->colo_cache, block->used_length); + block->colo_cache = NULL; + } + } + } + ram_state_cleanup(&ram_state); +} + +/** + * ram_load_setup: Setup RAM for migration incoming side + * + * Returns zero to indicate success and negative for error + * + * @f: QEMUFile where to receive the data + * @opaque: RAMState pointer + */ +static int ram_load_setup(QEMUFile *f, void *opaque) +{ + if (compress_threads_load_setup(f)) { + return -1; + } + + xbzrle_load_setup(); + ramblock_recv_map_init(); + + return 0; +} + +static int ram_load_cleanup(void *opaque) +{ + RAMBlock *rb; + + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { + qemu_ram_block_writeback(rb); + } + + xbzrle_load_cleanup(); + compress_threads_load_cleanup(); + + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { + g_free(rb->receivedmap); + rb->receivedmap = NULL; + } + + return 0; +} + +/** + * ram_postcopy_incoming_init: allocate postcopy data structures + * + * Returns 0 for success and negative if there was one error + * + * @mis: current migration incoming state + * + * Allocate data structures etc needed by incoming migration with + * postcopy-ram. postcopy-ram's similarly names + * postcopy_ram_incoming_init does the work. + */ +int ram_postcopy_incoming_init(MigrationIncomingState *mis) +{ + return postcopy_ram_incoming_init(mis); +} + +/** + * ram_load_postcopy: load a page in postcopy case + * + * Returns 0 for success or -errno in case of error + * + * Called in postcopy mode by ram_load(). + * rcu_read_lock is taken prior to this being called. + * + * @f: QEMUFile where to send the data + */ +static int ram_load_postcopy(QEMUFile *f) +{ + int flags = 0, ret = 0; + bool place_needed = false; + bool matches_target_page_size = false; + MigrationIncomingState *mis = migration_incoming_get_current(); + /* Temporary page that is later 'placed' */ + void *postcopy_host_page = mis->postcopy_tmp_page; + void *host_page = NULL; + bool all_zero = true; + int target_pages = 0; + + while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) { + ram_addr_t addr; + void *page_buffer = NULL; + void *place_source = NULL; + RAMBlock *block = NULL; + uint8_t ch; + int len; + + addr = qemu_get_be64(f); + + /* + * If qemu file error, we should stop here, and then "addr" + * may be invalid + */ + ret = qemu_file_get_error(f); + if (ret) { + break; + } + + flags = addr & ~TARGET_PAGE_MASK; + addr &= TARGET_PAGE_MASK; + + trace_ram_load_postcopy_loop((uint64_t)addr, flags); + if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE | + RAM_SAVE_FLAG_COMPRESS_PAGE)) { + block = ram_block_from_stream(f, flags); + if (!block) { + ret = -EINVAL; + break; + } + + /* + * Relying on used_length is racy and can result in false positives. + * We might place pages beyond used_length in case RAM was shrunk + * while in postcopy, which is fine - trying to place via + * UFFDIO_COPY/UFFDIO_ZEROPAGE will never segfault. + */ + if (!block->host || addr >= block->postcopy_length) { + error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); + ret = -EINVAL; + break; + } + target_pages++; + matches_target_page_size = block->page_size == TARGET_PAGE_SIZE; + /* + * Postcopy requires that we place whole host pages atomically; + * these may be huge pages for RAMBlocks that are backed by + * hugetlbfs. + * To make it atomic, the data is read into a temporary page + * that's moved into place later. + * The migration protocol uses, possibly smaller, target-pages + * however the source ensures it always sends all the components + * of a host page in one chunk. + */ + page_buffer = postcopy_host_page + + host_page_offset_from_ram_block_offset(block, addr); + /* If all TP are zero then we can optimise the place */ + if (target_pages == 1) { + host_page = host_page_from_ram_block_offset(block, addr); + } else if (host_page != host_page_from_ram_block_offset(block, + addr)) { + /* not the 1st TP within the HP */ + error_report("Non-same host page %p/%p", host_page, + host_page_from_ram_block_offset(block, addr)); + ret = -EINVAL; + break; + } + + /* + * If it's the last part of a host page then we place the host + * page + */ + if (target_pages == (block->page_size / TARGET_PAGE_SIZE)) { + place_needed = true; + } + place_source = postcopy_host_page; + } + + switch (flags & ~RAM_SAVE_FLAG_CONTINUE) { + case RAM_SAVE_FLAG_ZERO: + ch = qemu_get_byte(f); + /* + * Can skip to set page_buffer when + * this is a zero page and (block->page_size == TARGET_PAGE_SIZE). + */ + if (ch || !matches_target_page_size) { + memset(page_buffer, ch, TARGET_PAGE_SIZE); + } + if (ch) { + all_zero = false; + } + break; + + case RAM_SAVE_FLAG_PAGE: + all_zero = false; + if (!matches_target_page_size) { + /* For huge pages, we always use temporary buffer */ + qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE); + } else { + /* + * For small pages that matches target page size, we + * avoid the qemu_file copy. Instead we directly use + * the buffer of QEMUFile to place the page. Note: we + * cannot do any QEMUFile operation before using that + * buffer to make sure the buffer is valid when + * placing the page. + */ + qemu_get_buffer_in_place(f, (uint8_t **)&place_source, + TARGET_PAGE_SIZE); + } + break; + case RAM_SAVE_FLAG_COMPRESS_PAGE: + all_zero = false; + len = qemu_get_be32(f); + if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) { + error_report("Invalid compressed data length: %d", len); + ret = -EINVAL; + break; + } + decompress_data_with_multi_threads(f, page_buffer, len); + break; + + case RAM_SAVE_FLAG_EOS: + /* normal exit */ + multifd_recv_sync_main(); + break; + default: + error_report("Unknown combination of migration flags: 0x%x" + " (postcopy mode)", flags); + ret = -EINVAL; + break; + } + + /* Got the whole host page, wait for decompress before placing. */ + if (place_needed) { + ret |= wait_for_decompress_done(); + } + + /* Detect for any possible file errors */ + if (!ret && qemu_file_get_error(f)) { + ret = qemu_file_get_error(f); + } + + if (!ret && place_needed) { + if (all_zero) { + ret = postcopy_place_page_zero(mis, host_page, block); + } else { + ret = postcopy_place_page(mis, host_page, place_source, + block); + } + place_needed = false; + target_pages = 0; + /* Assume we have a zero page until we detect something different */ + all_zero = true; + } + } + + return ret; +} + +static bool postcopy_is_advised(void) +{ + PostcopyState ps = postcopy_state_get(); + return ps >= POSTCOPY_INCOMING_ADVISE && ps < POSTCOPY_INCOMING_END; +} + +static bool postcopy_is_running(void) +{ + PostcopyState ps = postcopy_state_get(); + return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END; +} + +/* + * Flush content of RAM cache into SVM's memory. + * Only flush the pages that be dirtied by PVM or SVM or both. + */ +void colo_flush_ram_cache(void) +{ + RAMBlock *block = NULL; + void *dst_host; + void *src_host; + unsigned long offset = 0; + + memory_global_dirty_log_sync(); + qemu_mutex_lock(&ram_state->bitmap_mutex); + WITH_RCU_READ_LOCK_GUARD() { + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + ramblock_sync_dirty_bitmap(ram_state, block); + } + } + + trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages); + WITH_RCU_READ_LOCK_GUARD() { + block = QLIST_FIRST_RCU(&ram_list.blocks); + + while (block) { + unsigned long num = 0; + + offset = colo_bitmap_find_dirty(ram_state, block, offset, &num); + if (!offset_in_ramblock(block, + ((ram_addr_t)offset) << TARGET_PAGE_BITS)) { + offset = 0; + num = 0; + block = QLIST_NEXT_RCU(block, next); + } else { + unsigned long i = 0; + + for (i = 0; i < num; i++) { + migration_bitmap_clear_dirty(ram_state, block, offset + i); + } + dst_host = block->host + + (((ram_addr_t)offset) << TARGET_PAGE_BITS); + src_host = block->colo_cache + + (((ram_addr_t)offset) << TARGET_PAGE_BITS); + memcpy(dst_host, src_host, TARGET_PAGE_SIZE * num); + offset += num; + } + } + } + trace_colo_flush_ram_cache_end(); + qemu_mutex_unlock(&ram_state->bitmap_mutex); +} + +/** + * ram_load_precopy: load pages in precopy case + * + * Returns 0 for success or -errno in case of error + * + * Called in precopy mode by ram_load(). + * rcu_read_lock is taken prior to this being called. + * + * @f: QEMUFile where to send the data + */ +static int ram_load_precopy(QEMUFile *f) +{ + int flags = 0, ret = 0, invalid_flags = 0, len = 0, i = 0; + /* ADVISE is earlier, it shows the source has the postcopy capability on */ + bool postcopy_advised = postcopy_is_advised(); + if (!migrate_use_compression()) { + invalid_flags |= RAM_SAVE_FLAG_COMPRESS_PAGE; + } + + while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) { + ram_addr_t addr, total_ram_bytes; + void *host = NULL, *host_bak = NULL; + uint8_t ch; + + /* + * Yield periodically to let main loop run, but an iteration of + * the main loop is expensive, so do it each some iterations + */ + if ((i & 32767) == 0 && qemu_in_coroutine()) { + aio_co_schedule(qemu_get_current_aio_context(), + qemu_coroutine_self()); + qemu_coroutine_yield(); + } + i++; + + addr = qemu_get_be64(f); + flags = addr & ~TARGET_PAGE_MASK; + addr &= TARGET_PAGE_MASK; + + if (flags & invalid_flags) { + if (flags & invalid_flags & RAM_SAVE_FLAG_COMPRESS_PAGE) { + error_report("Received an unexpected compressed page"); + } + + ret = -EINVAL; + break; + } + + if (flags & (RAM_SAVE_FLAG_ZERO | RAM_SAVE_FLAG_PAGE | + RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) { + RAMBlock *block = ram_block_from_stream(f, flags); + + host = host_from_ram_block_offset(block, addr); + /* + * After going into COLO stage, we should not load the page + * into SVM's memory directly, we put them into colo_cache firstly. + * NOTE: We need to keep a copy of SVM's ram in colo_cache. + * Previously, we copied all these memory in preparing stage of COLO + * while we need to stop VM, which is a time-consuming process. + * Here we optimize it by a trick, back-up every page while in + * migration process while COLO is enabled, though it affects the + * speed of the migration, but it obviously reduce the downtime of + * back-up all SVM'S memory in COLO preparing stage. + */ + if (migration_incoming_colo_enabled()) { + if (migration_incoming_in_colo_state()) { + /* In COLO stage, put all pages into cache temporarily */ + host = colo_cache_from_block_offset(block, addr, true); + } else { + /* + * In migration stage but before COLO stage, + * Put all pages into both cache and SVM's memory. + */ + host_bak = colo_cache_from_block_offset(block, addr, false); + } + } + if (!host) { + error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); + ret = -EINVAL; + break; + } + if (!migration_incoming_in_colo_state()) { + ramblock_recv_bitmap_set(block, host); + } + + trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host); + } + + switch (flags & ~RAM_SAVE_FLAG_CONTINUE) { + case RAM_SAVE_FLAG_MEM_SIZE: + /* Synchronize RAM block list */ + total_ram_bytes = addr; + while (!ret && total_ram_bytes) { + RAMBlock *block; + char id[256]; + ram_addr_t length; + + len = qemu_get_byte(f); + qemu_get_buffer(f, (uint8_t *)id, len); + id[len] = 0; + length = qemu_get_be64(f); + + block = qemu_ram_block_by_name(id); + if (block && !qemu_ram_is_migratable(block)) { + error_report("block %s should not be migrated !", id); + ret = -EINVAL; + } else if (block) { + if (length != block->used_length) { + Error *local_err = NULL; + + ret = qemu_ram_resize(block, length, + &local_err); + if (local_err) { + error_report_err(local_err); + } + } + /* For postcopy we need to check hugepage sizes match */ + if (postcopy_advised && migrate_postcopy_ram() && + block->page_size != qemu_host_page_size) { + uint64_t remote_page_size = qemu_get_be64(f); + if (remote_page_size != block->page_size) { + error_report("Mismatched RAM page size %s " + "(local) %zd != %" PRId64, + id, block->page_size, + remote_page_size); + ret = -EINVAL; + } + } + if (migrate_ignore_shared()) { + hwaddr addr = qemu_get_be64(f); + if (ramblock_is_ignored(block) && + block->mr->addr != addr) { + error_report("Mismatched GPAs for block %s " + "%" PRId64 "!= %" PRId64, + id, (uint64_t)addr, + (uint64_t)block->mr->addr); + ret = -EINVAL; + } + } + ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG, + block->idstr); + } else { + error_report("Unknown ramblock \"%s\", cannot " + "accept migration", id); + ret = -EINVAL; + } + + total_ram_bytes -= length; + } + break; + + case RAM_SAVE_FLAG_ZERO: + ch = qemu_get_byte(f); + ram_handle_compressed(host, ch, TARGET_PAGE_SIZE); + break; + + case RAM_SAVE_FLAG_PAGE: + qemu_get_buffer(f, host, TARGET_PAGE_SIZE); + break; + + case RAM_SAVE_FLAG_COMPRESS_PAGE: + len = qemu_get_be32(f); + if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) { + error_report("Invalid compressed data length: %d", len); + ret = -EINVAL; + break; + } + decompress_data_with_multi_threads(f, host, len); + break; + + case RAM_SAVE_FLAG_XBZRLE: + if (load_xbzrle(f, addr, host) < 0) { + error_report("Failed to decompress XBZRLE page at " + RAM_ADDR_FMT, addr); + ret = -EINVAL; + break; + } + break; + case RAM_SAVE_FLAG_EOS: + /* normal exit */ + multifd_recv_sync_main(); + break; + default: + if (flags & RAM_SAVE_FLAG_HOOK) { + ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL); + } else { + error_report("Unknown combination of migration flags: 0x%x", + flags); + ret = -EINVAL; + } + } + if (!ret) { + ret = qemu_file_get_error(f); + } + if (!ret && host_bak) { + memcpy(host_bak, host, TARGET_PAGE_SIZE); + } + } + + ret |= wait_for_decompress_done(); + return ret; +} + +static int ram_load(QEMUFile *f, void *opaque, int version_id) +{ + int ret = 0; + static uint64_t seq_iter; + /* + * If system is running in postcopy mode, page inserts to host memory must + * be atomic + */ + bool postcopy_running = postcopy_is_running(); + + seq_iter++; + + if (version_id != 4) { + return -EINVAL; + } + + /* + * This RCU critical section can be very long running. + * When RCU reclaims in the code start to become numerous, + * it will be necessary to reduce the granularity of this + * critical section. + */ + WITH_RCU_READ_LOCK_GUARD() { + if (postcopy_running) { + ret = ram_load_postcopy(f); + } else { + ret = ram_load_precopy(f); + } + } + trace_ram_load_complete(ret, seq_iter); + + return ret; +} + +static bool ram_has_postcopy(void *opaque) +{ + RAMBlock *rb; + RAMBLOCK_FOREACH_NOT_IGNORED(rb) { + if (ramblock_is_pmem(rb)) { + info_report("Block: %s, host: %p is a nvdimm memory, postcopy" + "is not supported now!", rb->idstr, rb->host); + return false; + } + } + + return migrate_postcopy_ram(); +} + +/* Sync all the dirty bitmap with destination VM. */ +static int ram_dirty_bitmap_sync_all(MigrationState *s, RAMState *rs) +{ + RAMBlock *block; + QEMUFile *file = s->to_dst_file; + int ramblock_count = 0; + + trace_ram_dirty_bitmap_sync_start(); + + RAMBLOCK_FOREACH_NOT_IGNORED(block) { + qemu_savevm_send_recv_bitmap(file, block->idstr); + trace_ram_dirty_bitmap_request(block->idstr); + ramblock_count++; + } + + trace_ram_dirty_bitmap_sync_wait(); + + /* Wait until all the ramblocks' dirty bitmap synced */ + while (ramblock_count--) { + qemu_sem_wait(&s->rp_state.rp_sem); + } + + trace_ram_dirty_bitmap_sync_complete(); + + return 0; +} + +static void ram_dirty_bitmap_reload_notify(MigrationState *s) +{ + qemu_sem_post(&s->rp_state.rp_sem); +} + +/* + * Read the received bitmap, revert it as the initial dirty bitmap. + * This is only used when the postcopy migration is paused but wants + * to resume from a middle point. + */ +int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *block) +{ + int ret = -EINVAL; + /* from_dst_file is always valid because we're within rp_thread */ + QEMUFile *file = s->rp_state.from_dst_file; + unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS; + uint64_t local_size = DIV_ROUND_UP(nbits, 8); + uint64_t size, end_mark; + + trace_ram_dirty_bitmap_reload_begin(block->idstr); + + if (s->state != MIGRATION_STATUS_POSTCOPY_RECOVER) { + error_report("%s: incorrect state %s", __func__, + MigrationStatus_str(s->state)); + return -EINVAL; + } + + /* + * Note: see comments in ramblock_recv_bitmap_send() on why we + * need the endianness conversion, and the paddings. + */ + local_size = ROUND_UP(local_size, 8); + + /* Add paddings */ + le_bitmap = bitmap_new(nbits + BITS_PER_LONG); + + size = qemu_get_be64(file); + + /* The size of the bitmap should match with our ramblock */ + if (size != local_size) { + error_report("%s: ramblock '%s' bitmap size mismatch " + "(0x%"PRIx64" != 0x%"PRIx64")", __func__, + block->idstr, size, local_size); + ret = -EINVAL; + goto out; + } + + size = qemu_get_buffer(file, (uint8_t *)le_bitmap, local_size); + end_mark = qemu_get_be64(file); + + ret = qemu_file_get_error(file); + if (ret || size != local_size) { + error_report("%s: read bitmap failed for ramblock '%s': %d" + " (size 0x%"PRIx64", got: 0x%"PRIx64")", + __func__, block->idstr, ret, local_size, size); + ret = -EIO; + goto out; + } + + if (end_mark != RAMBLOCK_RECV_BITMAP_ENDING) { + error_report("%s: ramblock '%s' end mark incorrect: 0x%"PRIx64, + __func__, block->idstr, end_mark); + ret = -EINVAL; + goto out; + } + + /* + * Endianness conversion. We are during postcopy (though paused). + * The dirty bitmap won't change. We can directly modify it. + */ + bitmap_from_le(block->bmap, le_bitmap, nbits); + + /* + * What we received is "received bitmap". Revert it as the initial + * dirty bitmap for this ramblock. + */ + bitmap_complement(block->bmap, block->bmap, nbits); + + /* Clear dirty bits of discarded ranges that we don't want to migrate. */ + ramblock_dirty_bitmap_clear_discarded_pages(block); + + /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). */ + trace_ram_dirty_bitmap_reload_complete(block->idstr); + + /* + * We succeeded to sync bitmap for current ramblock. If this is + * the last one to sync, we need to notify the main send thread. + */ + ram_dirty_bitmap_reload_notify(s); + + ret = 0; +out: + g_free(le_bitmap); + return ret; +} + +static int ram_resume_prepare(MigrationState *s, void *opaque) +{ + RAMState *rs = *(RAMState **)opaque; + int ret; + + ret = ram_dirty_bitmap_sync_all(s, rs); + if (ret) { + return ret; + } + + ram_state_resume_prepare(rs, s->to_dst_file); + + return 0; +} + +static SaveVMHandlers savevm_ram_handlers = { + .save_setup = ram_save_setup, + .save_live_iterate = ram_save_iterate, + .save_live_complete_postcopy = ram_save_complete, + .save_live_complete_precopy = ram_save_complete, + .has_postcopy = ram_has_postcopy, + .save_live_pending = ram_save_pending, + .load_state = ram_load, + .save_cleanup = ram_save_cleanup, + .load_setup = ram_load_setup, + .load_cleanup = ram_load_cleanup, + .resume_prepare = ram_resume_prepare, +}; + +static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host, + size_t old_size, size_t new_size) +{ + PostcopyState ps = postcopy_state_get(); + ram_addr_t offset; + RAMBlock *rb = qemu_ram_block_from_host(host, false, &offset); + Error *err = NULL; + + if (ramblock_is_ignored(rb)) { + return; + } + + if (!migration_is_idle()) { + /* + * Precopy code on the source cannot deal with the size of RAM blocks + * changing at random points in time - especially after sending the + * RAM block sizes in the migration stream, they must no longer change. + * Abort and indicate a proper reason. + */ + error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr); + migration_cancel(err); + error_free(err); + } + + switch (ps) { + case POSTCOPY_INCOMING_ADVISE: + /* + * Update what ram_postcopy_incoming_init()->init_range() does at the + * time postcopy was advised. Syncing RAM blocks with the source will + * result in RAM resizes. + */ + if (old_size < new_size) { + if (ram_discard_range(rb->idstr, old_size, new_size - old_size)) { + error_report("RAM block '%s' discard of resized RAM failed", + rb->idstr); + } + } + rb->postcopy_length = new_size; + break; + case POSTCOPY_INCOMING_NONE: + case POSTCOPY_INCOMING_RUNNING: + case POSTCOPY_INCOMING_END: + /* + * Once our guest is running, postcopy does no longer care about + * resizes. When growing, the new memory was not available on the + * source, no handler needed. + */ + break; + default: + error_report("RAM block '%s' resized during postcopy state: %d", + rb->idstr, ps); + exit(-1); + } +} + +static RAMBlockNotifier ram_mig_ram_notifier = { + .ram_block_resized = ram_mig_ram_block_resized, +}; + +void ram_mig_init(void) +{ + qemu_mutex_init(&XBZRLE.lock); + register_savevm_live("ram", 0, 4, &savevm_ram_handlers, &ram_state); + ram_block_notifier_add(&ram_mig_ram_notifier); +} diff --git a/migration/ram.h b/migration/ram.h new file mode 100644 index 000000000..c515396a9 --- /dev/null +++ b/migration/ram.h @@ -0,0 +1,91 @@ +/* + * QEMU System Emulator + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2011-2015 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef QEMU_MIGRATION_RAM_H +#define QEMU_MIGRATION_RAM_H + +#include "qapi/qapi-types-migration.h" +#include "exec/cpu-common.h" +#include "io/channel.h" + +extern MigrationStats ram_counters; +extern XBZRLECacheStats xbzrle_counters; +extern CompressionStats compression_counters; + +bool ramblock_is_ignored(RAMBlock *block); +/* Should be holding either ram_list.mutex, or the RCU lock. */ +#define RAMBLOCK_FOREACH_NOT_IGNORED(block) \ + INTERNAL_RAMBLOCK_FOREACH(block) \ + if (ramblock_is_ignored(block)) {} else + +#define RAMBLOCK_FOREACH_MIGRATABLE(block) \ + INTERNAL_RAMBLOCK_FOREACH(block) \ + if (!qemu_ram_is_migratable(block)) {} else + +int xbzrle_cache_resize(uint64_t new_size, Error **errp); +uint64_t ram_bytes_remaining(void); +uint64_t ram_bytes_total(void); +void mig_throttle_counter_reset(void); + +uint64_t ram_pagesize_summary(void); +int ram_save_queue_pages(const char *rbname, ram_addr_t start, ram_addr_t len); +void acct_update_position(QEMUFile *f, size_t size, bool zero); +void ram_debug_dump_bitmap(unsigned long *todump, bool expected, + unsigned long pages); +void ram_postcopy_migrated_memory_release(MigrationState *ms); +/* For outgoing discard bitmap */ +int ram_postcopy_send_discard_bitmap(MigrationState *ms); +/* For incoming postcopy discard */ +int ram_discard_range(const char *block_name, uint64_t start, size_t length); +int ram_postcopy_incoming_init(MigrationIncomingState *mis); + +void ram_handle_compressed(void *host, uint8_t ch, uint64_t size); + +int ramblock_recv_bitmap_test(RAMBlock *rb, void *host_addr); +bool ramblock_recv_bitmap_test_byte_offset(RAMBlock *rb, uint64_t byte_offset); +void ramblock_recv_bitmap_set(RAMBlock *rb, void *host_addr); +void ramblock_recv_bitmap_set_range(RAMBlock *rb, void *host_addr, size_t nr); +int64_t ramblock_recv_bitmap_send(QEMUFile *file, + const char *block_name); +int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb); +bool ramblock_page_is_discarded(RAMBlock *rb, ram_addr_t start); + +/* ram cache */ +int colo_init_ram_cache(void); +void colo_flush_ram_cache(void); +void colo_release_ram_cache(void); +void colo_incoming_start_dirty_log(void); + +/* Background snapshot */ +bool ram_write_tracking_available(void); +bool ram_write_tracking_compatible(void); +void ram_write_tracking_prepare(void); +int ram_write_tracking_start(void); +void ram_write_tracking_stop(void); + +#endif diff --git a/migration/rdma.c b/migration/rdma.c new file mode 100644 index 000000000..f5d3bbe7e --- /dev/null +++ b/migration/rdma.c @@ -0,0 +1,4335 @@ +/* + * RDMA protocol and interfaces + * + * Copyright IBM, Corp. 2010-2013 + * Copyright Red Hat, Inc. 2015-2016 + * + * Authors: + * Michael R. Hines <mrhines@us.ibm.com> + * Jiuxing Liu <jl@us.ibm.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + * + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "qemu/cutils.h" +#include "rdma.h" +#include "migration.h" +#include "qemu-file.h" +#include "ram.h" +#include "qemu-file-channel.h" +#include "qemu/error-report.h" +#include "qemu/main-loop.h" +#include "qemu/module.h" +#include "qemu/rcu.h" +#include "qemu/sockets.h" +#include "qemu/bitmap.h" +#include "qemu/coroutine.h" +#include "exec/memory.h" +#include <sys/socket.h> +#include <netdb.h> +#include <arpa/inet.h> +#include <rdma/rdma_cma.h> +#include "trace.h" +#include "qom/object.h" +#include <poll.h> + +/* + * Print and error on both the Monitor and the Log file. + */ +#define ERROR(errp, fmt, ...) \ + do { \ + fprintf(stderr, "RDMA ERROR: " fmt "\n", ## __VA_ARGS__); \ + if (errp && (*(errp) == NULL)) { \ + error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \ + } \ + } while (0) + +#define RDMA_RESOLVE_TIMEOUT_MS 10000 + +/* Do not merge data if larger than this. */ +#define RDMA_MERGE_MAX (2 * 1024 * 1024) +#define RDMA_SIGNALED_SEND_MAX (RDMA_MERGE_MAX / 4096) + +#define RDMA_REG_CHUNK_SHIFT 20 /* 1 MB */ + +/* + * This is only for non-live state being migrated. + * Instead of RDMA_WRITE messages, we use RDMA_SEND + * messages for that state, which requires a different + * delivery design than main memory. + */ +#define RDMA_SEND_INCREMENT 32768 + +/* + * Maximum size infiniband SEND message + */ +#define RDMA_CONTROL_MAX_BUFFER (512 * 1024) +#define RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE 4096 + +#define RDMA_CONTROL_VERSION_CURRENT 1 +/* + * Capabilities for negotiation. + */ +#define RDMA_CAPABILITY_PIN_ALL 0x01 + +/* + * Add the other flags above to this list of known capabilities + * as they are introduced. + */ +static uint32_t known_capabilities = RDMA_CAPABILITY_PIN_ALL; + +#define CHECK_ERROR_STATE() \ + do { \ + if (rdma->error_state) { \ + if (!rdma->error_reported) { \ + error_report("RDMA is in an error state waiting migration" \ + " to abort!"); \ + rdma->error_reported = 1; \ + } \ + return rdma->error_state; \ + } \ + } while (0) + +/* + * A work request ID is 64-bits and we split up these bits + * into 3 parts: + * + * bits 0-15 : type of control message, 2^16 + * bits 16-29: ram block index, 2^14 + * bits 30-63: ram block chunk number, 2^34 + * + * The last two bit ranges are only used for RDMA writes, + * in order to track their completion and potentially + * also track unregistration status of the message. + */ +#define RDMA_WRID_TYPE_SHIFT 0UL +#define RDMA_WRID_BLOCK_SHIFT 16UL +#define RDMA_WRID_CHUNK_SHIFT 30UL + +#define RDMA_WRID_TYPE_MASK \ + ((1UL << RDMA_WRID_BLOCK_SHIFT) - 1UL) + +#define RDMA_WRID_BLOCK_MASK \ + (~RDMA_WRID_TYPE_MASK & ((1UL << RDMA_WRID_CHUNK_SHIFT) - 1UL)) + +#define RDMA_WRID_CHUNK_MASK (~RDMA_WRID_BLOCK_MASK & ~RDMA_WRID_TYPE_MASK) + +/* + * RDMA migration protocol: + * 1. RDMA Writes (data messages, i.e. RAM) + * 2. IB Send/Recv (control channel messages) + */ +enum { + RDMA_WRID_NONE = 0, + RDMA_WRID_RDMA_WRITE = 1, + RDMA_WRID_SEND_CONTROL = 2000, + RDMA_WRID_RECV_CONTROL = 4000, +}; + +static const char *wrid_desc[] = { + [RDMA_WRID_NONE] = "NONE", + [RDMA_WRID_RDMA_WRITE] = "WRITE RDMA", + [RDMA_WRID_SEND_CONTROL] = "CONTROL SEND", + [RDMA_WRID_RECV_CONTROL] = "CONTROL RECV", +}; + +/* + * Work request IDs for IB SEND messages only (not RDMA writes). + * This is used by the migration protocol to transmit + * control messages (such as device state and registration commands) + * + * We could use more WRs, but we have enough for now. + */ +enum { + RDMA_WRID_READY = 0, + RDMA_WRID_DATA, + RDMA_WRID_CONTROL, + RDMA_WRID_MAX, +}; + +/* + * SEND/RECV IB Control Messages. + */ +enum { + RDMA_CONTROL_NONE = 0, + RDMA_CONTROL_ERROR, + RDMA_CONTROL_READY, /* ready to receive */ + RDMA_CONTROL_QEMU_FILE, /* QEMUFile-transmitted bytes */ + RDMA_CONTROL_RAM_BLOCKS_REQUEST, /* RAMBlock synchronization */ + RDMA_CONTROL_RAM_BLOCKS_RESULT, /* RAMBlock synchronization */ + RDMA_CONTROL_COMPRESS, /* page contains repeat values */ + RDMA_CONTROL_REGISTER_REQUEST, /* dynamic page registration */ + RDMA_CONTROL_REGISTER_RESULT, /* key to use after registration */ + RDMA_CONTROL_REGISTER_FINISHED, /* current iteration finished */ + RDMA_CONTROL_UNREGISTER_REQUEST, /* dynamic UN-registration */ + RDMA_CONTROL_UNREGISTER_FINISHED, /* unpinning finished */ +}; + + +/* + * Memory and MR structures used to represent an IB Send/Recv work request. + * This is *not* used for RDMA writes, only IB Send/Recv. + */ +typedef struct { + uint8_t control[RDMA_CONTROL_MAX_BUFFER]; /* actual buffer to register */ + struct ibv_mr *control_mr; /* registration metadata */ + size_t control_len; /* length of the message */ + uint8_t *control_curr; /* start of unconsumed bytes */ +} RDMAWorkRequestData; + +/* + * Negotiate RDMA capabilities during connection-setup time. + */ +typedef struct { + uint32_t version; + uint32_t flags; +} RDMACapabilities; + +static void caps_to_network(RDMACapabilities *cap) +{ + cap->version = htonl(cap->version); + cap->flags = htonl(cap->flags); +} + +static void network_to_caps(RDMACapabilities *cap) +{ + cap->version = ntohl(cap->version); + cap->flags = ntohl(cap->flags); +} + +/* + * Representation of a RAMBlock from an RDMA perspective. + * This is not transmitted, only local. + * This and subsequent structures cannot be linked lists + * because we're using a single IB message to transmit + * the information. It's small anyway, so a list is overkill. + */ +typedef struct RDMALocalBlock { + char *block_name; + uint8_t *local_host_addr; /* local virtual address */ + uint64_t remote_host_addr; /* remote virtual address */ + uint64_t offset; + uint64_t length; + struct ibv_mr **pmr; /* MRs for chunk-level registration */ + struct ibv_mr *mr; /* MR for non-chunk-level registration */ + uint32_t *remote_keys; /* rkeys for chunk-level registration */ + uint32_t remote_rkey; /* rkeys for non-chunk-level registration */ + int index; /* which block are we */ + unsigned int src_index; /* (Only used on dest) */ + bool is_ram_block; + int nb_chunks; + unsigned long *transit_bitmap; + unsigned long *unregister_bitmap; +} RDMALocalBlock; + +/* + * Also represents a RAMblock, but only on the dest. + * This gets transmitted by the dest during connection-time + * to the source VM and then is used to populate the + * corresponding RDMALocalBlock with + * the information needed to perform the actual RDMA. + */ +typedef struct QEMU_PACKED RDMADestBlock { + uint64_t remote_host_addr; + uint64_t offset; + uint64_t length; + uint32_t remote_rkey; + uint32_t padding; +} RDMADestBlock; + +static const char *control_desc(unsigned int rdma_control) +{ + static const char *strs[] = { + [RDMA_CONTROL_NONE] = "NONE", + [RDMA_CONTROL_ERROR] = "ERROR", + [RDMA_CONTROL_READY] = "READY", + [RDMA_CONTROL_QEMU_FILE] = "QEMU FILE", + [RDMA_CONTROL_RAM_BLOCKS_REQUEST] = "RAM BLOCKS REQUEST", + [RDMA_CONTROL_RAM_BLOCKS_RESULT] = "RAM BLOCKS RESULT", + [RDMA_CONTROL_COMPRESS] = "COMPRESS", + [RDMA_CONTROL_REGISTER_REQUEST] = "REGISTER REQUEST", + [RDMA_CONTROL_REGISTER_RESULT] = "REGISTER RESULT", + [RDMA_CONTROL_REGISTER_FINISHED] = "REGISTER FINISHED", + [RDMA_CONTROL_UNREGISTER_REQUEST] = "UNREGISTER REQUEST", + [RDMA_CONTROL_UNREGISTER_FINISHED] = "UNREGISTER FINISHED", + }; + + if (rdma_control > RDMA_CONTROL_UNREGISTER_FINISHED) { + return "??BAD CONTROL VALUE??"; + } + + return strs[rdma_control]; +} + +static uint64_t htonll(uint64_t v) +{ + union { uint32_t lv[2]; uint64_t llv; } u; + u.lv[0] = htonl(v >> 32); + u.lv[1] = htonl(v & 0xFFFFFFFFULL); + return u.llv; +} + +static uint64_t ntohll(uint64_t v) +{ + union { uint32_t lv[2]; uint64_t llv; } u; + u.llv = v; + return ((uint64_t)ntohl(u.lv[0]) << 32) | (uint64_t) ntohl(u.lv[1]); +} + +static void dest_block_to_network(RDMADestBlock *db) +{ + db->remote_host_addr = htonll(db->remote_host_addr); + db->offset = htonll(db->offset); + db->length = htonll(db->length); + db->remote_rkey = htonl(db->remote_rkey); +} + +static void network_to_dest_block(RDMADestBlock *db) +{ + db->remote_host_addr = ntohll(db->remote_host_addr); + db->offset = ntohll(db->offset); + db->length = ntohll(db->length); + db->remote_rkey = ntohl(db->remote_rkey); +} + +/* + * Virtual address of the above structures used for transmitting + * the RAMBlock descriptions at connection-time. + * This structure is *not* transmitted. + */ +typedef struct RDMALocalBlocks { + int nb_blocks; + bool init; /* main memory init complete */ + RDMALocalBlock *block; +} RDMALocalBlocks; + +/* + * Main data structure for RDMA state. + * While there is only one copy of this structure being allocated right now, + * this is the place where one would start if you wanted to consider + * having more than one RDMA connection open at the same time. + */ +typedef struct RDMAContext { + char *host; + int port; + char *host_port; + + RDMAWorkRequestData wr_data[RDMA_WRID_MAX]; + + /* + * This is used by *_exchange_send() to figure out whether or not + * the initial "READY" message has already been received or not. + * This is because other functions may potentially poll() and detect + * the READY message before send() does, in which case we need to + * know if it completed. + */ + int control_ready_expected; + + /* number of outstanding writes */ + int nb_sent; + + /* store info about current buffer so that we can + merge it with future sends */ + uint64_t current_addr; + uint64_t current_length; + /* index of ram block the current buffer belongs to */ + int current_index; + /* index of the chunk in the current ram block */ + int current_chunk; + + bool pin_all; + + /* + * infiniband-specific variables for opening the device + * and maintaining connection state and so forth. + * + * cm_id also has ibv_context, rdma_event_channel, and ibv_qp in + * cm_id->verbs, cm_id->channel, and cm_id->qp. + */ + struct rdma_cm_id *cm_id; /* connection manager ID */ + struct rdma_cm_id *listen_id; + bool connected; + + struct ibv_context *verbs; + struct rdma_event_channel *channel; + struct ibv_qp *qp; /* queue pair */ + struct ibv_comp_channel *recv_comp_channel; /* recv completion channel */ + struct ibv_comp_channel *send_comp_channel; /* send completion channel */ + struct ibv_pd *pd; /* protection domain */ + struct ibv_cq *recv_cq; /* recvieve completion queue */ + struct ibv_cq *send_cq; /* send completion queue */ + + /* + * If a previous write failed (perhaps because of a failed + * memory registration, then do not attempt any future work + * and remember the error state. + */ + int error_state; + int error_reported; + int received_error; + + /* + * Description of ram blocks used throughout the code. + */ + RDMALocalBlocks local_ram_blocks; + RDMADestBlock *dest_blocks; + + /* Index of the next RAMBlock received during block registration */ + unsigned int next_src_index; + + /* + * Migration on *destination* started. + * Then use coroutine yield function. + * Source runs in a thread, so we don't care. + */ + int migration_started_on_destination; + + int total_registrations; + int total_writes; + + int unregister_current, unregister_next; + uint64_t unregistrations[RDMA_SIGNALED_SEND_MAX]; + + GHashTable *blockmap; + + /* the RDMAContext for return path */ + struct RDMAContext *return_path; + bool is_return_path; +} RDMAContext; + +#define TYPE_QIO_CHANNEL_RDMA "qio-channel-rdma" +OBJECT_DECLARE_SIMPLE_TYPE(QIOChannelRDMA, QIO_CHANNEL_RDMA) + + + +struct QIOChannelRDMA { + QIOChannel parent; + RDMAContext *rdmain; + RDMAContext *rdmaout; + QEMUFile *file; + bool blocking; /* XXX we don't actually honour this yet */ +}; + +/* + * Main structure for IB Send/Recv control messages. + * This gets prepended at the beginning of every Send/Recv. + */ +typedef struct QEMU_PACKED { + uint32_t len; /* Total length of data portion */ + uint32_t type; /* which control command to perform */ + uint32_t repeat; /* number of commands in data portion of same type */ + uint32_t padding; +} RDMAControlHeader; + +static void control_to_network(RDMAControlHeader *control) +{ + control->type = htonl(control->type); + control->len = htonl(control->len); + control->repeat = htonl(control->repeat); +} + +static void network_to_control(RDMAControlHeader *control) +{ + control->type = ntohl(control->type); + control->len = ntohl(control->len); + control->repeat = ntohl(control->repeat); +} + +/* + * Register a single Chunk. + * Information sent by the source VM to inform the dest + * to register an single chunk of memory before we can perform + * the actual RDMA operation. + */ +typedef struct QEMU_PACKED { + union QEMU_PACKED { + uint64_t current_addr; /* offset into the ram_addr_t space */ + uint64_t chunk; /* chunk to lookup if unregistering */ + } key; + uint32_t current_index; /* which ramblock the chunk belongs to */ + uint32_t padding; + uint64_t chunks; /* how many sequential chunks to register */ +} RDMARegister; + +static void register_to_network(RDMAContext *rdma, RDMARegister *reg) +{ + RDMALocalBlock *local_block; + local_block = &rdma->local_ram_blocks.block[reg->current_index]; + + if (local_block->is_ram_block) { + /* + * current_addr as passed in is an address in the local ram_addr_t + * space, we need to translate this for the destination + */ + reg->key.current_addr -= local_block->offset; + reg->key.current_addr += rdma->dest_blocks[reg->current_index].offset; + } + reg->key.current_addr = htonll(reg->key.current_addr); + reg->current_index = htonl(reg->current_index); + reg->chunks = htonll(reg->chunks); +} + +static void network_to_register(RDMARegister *reg) +{ + reg->key.current_addr = ntohll(reg->key.current_addr); + reg->current_index = ntohl(reg->current_index); + reg->chunks = ntohll(reg->chunks); +} + +typedef struct QEMU_PACKED { + uint32_t value; /* if zero, we will madvise() */ + uint32_t block_idx; /* which ram block index */ + uint64_t offset; /* Address in remote ram_addr_t space */ + uint64_t length; /* length of the chunk */ +} RDMACompress; + +static void compress_to_network(RDMAContext *rdma, RDMACompress *comp) +{ + comp->value = htonl(comp->value); + /* + * comp->offset as passed in is an address in the local ram_addr_t + * space, we need to translate this for the destination + */ + comp->offset -= rdma->local_ram_blocks.block[comp->block_idx].offset; + comp->offset += rdma->dest_blocks[comp->block_idx].offset; + comp->block_idx = htonl(comp->block_idx); + comp->offset = htonll(comp->offset); + comp->length = htonll(comp->length); +} + +static void network_to_compress(RDMACompress *comp) +{ + comp->value = ntohl(comp->value); + comp->block_idx = ntohl(comp->block_idx); + comp->offset = ntohll(comp->offset); + comp->length = ntohll(comp->length); +} + +/* + * The result of the dest's memory registration produces an "rkey" + * which the source VM must reference in order to perform + * the RDMA operation. + */ +typedef struct QEMU_PACKED { + uint32_t rkey; + uint32_t padding; + uint64_t host_addr; +} RDMARegisterResult; + +static void result_to_network(RDMARegisterResult *result) +{ + result->rkey = htonl(result->rkey); + result->host_addr = htonll(result->host_addr); +}; + +static void network_to_result(RDMARegisterResult *result) +{ + result->rkey = ntohl(result->rkey); + result->host_addr = ntohll(result->host_addr); +}; + +const char *print_wrid(int wrid); +static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head, + uint8_t *data, RDMAControlHeader *resp, + int *resp_idx, + int (*callback)(RDMAContext *rdma)); + +static inline uint64_t ram_chunk_index(const uint8_t *start, + const uint8_t *host) +{ + return ((uintptr_t) host - (uintptr_t) start) >> RDMA_REG_CHUNK_SHIFT; +} + +static inline uint8_t *ram_chunk_start(const RDMALocalBlock *rdma_ram_block, + uint64_t i) +{ + return (uint8_t *)(uintptr_t)(rdma_ram_block->local_host_addr + + (i << RDMA_REG_CHUNK_SHIFT)); +} + +static inline uint8_t *ram_chunk_end(const RDMALocalBlock *rdma_ram_block, + uint64_t i) +{ + uint8_t *result = ram_chunk_start(rdma_ram_block, i) + + (1UL << RDMA_REG_CHUNK_SHIFT); + + if (result > (rdma_ram_block->local_host_addr + rdma_ram_block->length)) { + result = rdma_ram_block->local_host_addr + rdma_ram_block->length; + } + + return result; +} + +static int rdma_add_block(RDMAContext *rdma, const char *block_name, + void *host_addr, + ram_addr_t block_offset, uint64_t length) +{ + RDMALocalBlocks *local = &rdma->local_ram_blocks; + RDMALocalBlock *block; + RDMALocalBlock *old = local->block; + + local->block = g_new0(RDMALocalBlock, local->nb_blocks + 1); + + if (local->nb_blocks) { + int x; + + if (rdma->blockmap) { + for (x = 0; x < local->nb_blocks; x++) { + g_hash_table_remove(rdma->blockmap, + (void *)(uintptr_t)old[x].offset); + g_hash_table_insert(rdma->blockmap, + (void *)(uintptr_t)old[x].offset, + &local->block[x]); + } + } + memcpy(local->block, old, sizeof(RDMALocalBlock) * local->nb_blocks); + g_free(old); + } + + block = &local->block[local->nb_blocks]; + + block->block_name = g_strdup(block_name); + block->local_host_addr = host_addr; + block->offset = block_offset; + block->length = length; + block->index = local->nb_blocks; + block->src_index = ~0U; /* Filled in by the receipt of the block list */ + block->nb_chunks = ram_chunk_index(host_addr, host_addr + length) + 1UL; + block->transit_bitmap = bitmap_new(block->nb_chunks); + bitmap_clear(block->transit_bitmap, 0, block->nb_chunks); + block->unregister_bitmap = bitmap_new(block->nb_chunks); + bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks); + block->remote_keys = g_new0(uint32_t, block->nb_chunks); + + block->is_ram_block = local->init ? false : true; + + if (rdma->blockmap) { + g_hash_table_insert(rdma->blockmap, (void *)(uintptr_t)block_offset, block); + } + + trace_rdma_add_block(block_name, local->nb_blocks, + (uintptr_t) block->local_host_addr, + block->offset, block->length, + (uintptr_t) (block->local_host_addr + block->length), + BITS_TO_LONGS(block->nb_chunks) * + sizeof(unsigned long) * 8, + block->nb_chunks); + + local->nb_blocks++; + + return 0; +} + +/* + * Memory regions need to be registered with the device and queue pairs setup + * in advanced before the migration starts. This tells us where the RAM blocks + * are so that we can register them individually. + */ +static int qemu_rdma_init_one_block(RAMBlock *rb, void *opaque) +{ + const char *block_name = qemu_ram_get_idstr(rb); + void *host_addr = qemu_ram_get_host_addr(rb); + ram_addr_t block_offset = qemu_ram_get_offset(rb); + ram_addr_t length = qemu_ram_get_used_length(rb); + return rdma_add_block(opaque, block_name, host_addr, block_offset, length); +} + +/* + * Identify the RAMBlocks and their quantity. They will be references to + * identify chunk boundaries inside each RAMBlock and also be referenced + * during dynamic page registration. + */ +static int qemu_rdma_init_ram_blocks(RDMAContext *rdma) +{ + RDMALocalBlocks *local = &rdma->local_ram_blocks; + int ret; + + assert(rdma->blockmap == NULL); + memset(local, 0, sizeof *local); + ret = foreach_not_ignored_block(qemu_rdma_init_one_block, rdma); + if (ret) { + return ret; + } + trace_qemu_rdma_init_ram_blocks(local->nb_blocks); + rdma->dest_blocks = g_new0(RDMADestBlock, + rdma->local_ram_blocks.nb_blocks); + local->init = true; + return 0; +} + +/* + * Note: If used outside of cleanup, the caller must ensure that the destination + * block structures are also updated + */ +static int rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block) +{ + RDMALocalBlocks *local = &rdma->local_ram_blocks; + RDMALocalBlock *old = local->block; + int x; + + if (rdma->blockmap) { + g_hash_table_remove(rdma->blockmap, (void *)(uintptr_t)block->offset); + } + if (block->pmr) { + int j; + + for (j = 0; j < block->nb_chunks; j++) { + if (!block->pmr[j]) { + continue; + } + ibv_dereg_mr(block->pmr[j]); + rdma->total_registrations--; + } + g_free(block->pmr); + block->pmr = NULL; + } + + if (block->mr) { + ibv_dereg_mr(block->mr); + rdma->total_registrations--; + block->mr = NULL; + } + + g_free(block->transit_bitmap); + block->transit_bitmap = NULL; + + g_free(block->unregister_bitmap); + block->unregister_bitmap = NULL; + + g_free(block->remote_keys); + block->remote_keys = NULL; + + g_free(block->block_name); + block->block_name = NULL; + + if (rdma->blockmap) { + for (x = 0; x < local->nb_blocks; x++) { + g_hash_table_remove(rdma->blockmap, + (void *)(uintptr_t)old[x].offset); + } + } + + if (local->nb_blocks > 1) { + + local->block = g_new0(RDMALocalBlock, local->nb_blocks - 1); + + if (block->index) { + memcpy(local->block, old, sizeof(RDMALocalBlock) * block->index); + } + + if (block->index < (local->nb_blocks - 1)) { + memcpy(local->block + block->index, old + (block->index + 1), + sizeof(RDMALocalBlock) * + (local->nb_blocks - (block->index + 1))); + for (x = block->index; x < local->nb_blocks - 1; x++) { + local->block[x].index--; + } + } + } else { + assert(block == local->block); + local->block = NULL; + } + + trace_rdma_delete_block(block, (uintptr_t)block->local_host_addr, + block->offset, block->length, + (uintptr_t)(block->local_host_addr + block->length), + BITS_TO_LONGS(block->nb_chunks) * + sizeof(unsigned long) * 8, block->nb_chunks); + + g_free(old); + + local->nb_blocks--; + + if (local->nb_blocks && rdma->blockmap) { + for (x = 0; x < local->nb_blocks; x++) { + g_hash_table_insert(rdma->blockmap, + (void *)(uintptr_t)local->block[x].offset, + &local->block[x]); + } + } + + return 0; +} + +/* + * Put in the log file which RDMA device was opened and the details + * associated with that device. + */ +static void qemu_rdma_dump_id(const char *who, struct ibv_context *verbs) +{ + struct ibv_port_attr port; + + if (ibv_query_port(verbs, 1, &port)) { + error_report("Failed to query port information"); + return; + } + + printf("%s RDMA Device opened: kernel name %s " + "uverbs device name %s, " + "infiniband_verbs class device path %s, " + "infiniband class device path %s, " + "transport: (%d) %s\n", + who, + verbs->device->name, + verbs->device->dev_name, + verbs->device->dev_path, + verbs->device->ibdev_path, + port.link_layer, + (port.link_layer == IBV_LINK_LAYER_INFINIBAND) ? "Infiniband" : + ((port.link_layer == IBV_LINK_LAYER_ETHERNET) + ? "Ethernet" : "Unknown")); +} + +/* + * Put in the log file the RDMA gid addressing information, + * useful for folks who have trouble understanding the + * RDMA device hierarchy in the kernel. + */ +static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id) +{ + char sgid[33]; + char dgid[33]; + inet_ntop(AF_INET6, &id->route.addr.addr.ibaddr.sgid, sgid, sizeof sgid); + inet_ntop(AF_INET6, &id->route.addr.addr.ibaddr.dgid, dgid, sizeof dgid); + trace_qemu_rdma_dump_gid(who, sgid, dgid); +} + +/* + * As of now, IPv6 over RoCE / iWARP is not supported by linux. + * We will try the next addrinfo struct, and fail if there are + * no other valid addresses to bind against. + * + * If user is listening on '[::]', then we will not have a opened a device + * yet and have no way of verifying if the device is RoCE or not. + * + * In this case, the source VM will throw an error for ALL types of + * connections (both IPv4 and IPv6) if the destination machine does not have + * a regular infiniband network available for use. + * + * The only way to guarantee that an error is thrown for broken kernels is + * for the management software to choose a *specific* interface at bind time + * and validate what time of hardware it is. + * + * Unfortunately, this puts the user in a fix: + * + * If the source VM connects with an IPv4 address without knowing that the + * destination has bound to '[::]' the migration will unconditionally fail + * unless the management software is explicitly listening on the IPv4 + * address while using a RoCE-based device. + * + * If the source VM connects with an IPv6 address, then we're OK because we can + * throw an error on the source (and similarly on the destination). + * + * But in mixed environments, this will be broken for a while until it is fixed + * inside linux. + * + * We do provide a *tiny* bit of help in this function: We can list all of the + * devices in the system and check to see if all the devices are RoCE or + * Infiniband. + * + * If we detect that we have a *pure* RoCE environment, then we can safely + * thrown an error even if the management software has specified '[::]' as the + * bind address. + * + * However, if there is are multiple hetergeneous devices, then we cannot make + * this assumption and the user just has to be sure they know what they are + * doing. + * + * Patches are being reviewed on linux-rdma. + */ +static int qemu_rdma_broken_ipv6_kernel(struct ibv_context *verbs, Error **errp) +{ + /* This bug only exists in linux, to our knowledge. */ +#ifdef CONFIG_LINUX + struct ibv_port_attr port_attr; + + /* + * Verbs are only NULL if management has bound to '[::]'. + * + * Let's iterate through all the devices and see if there any pure IB + * devices (non-ethernet). + * + * If not, then we can safely proceed with the migration. + * Otherwise, there are no guarantees until the bug is fixed in linux. + */ + if (!verbs) { + int num_devices, x; + struct ibv_device **dev_list = ibv_get_device_list(&num_devices); + bool roce_found = false; + bool ib_found = false; + + for (x = 0; x < num_devices; x++) { + verbs = ibv_open_device(dev_list[x]); + if (!verbs) { + if (errno == EPERM) { + continue; + } else { + return -EINVAL; + } + } + + if (ibv_query_port(verbs, 1, &port_attr)) { + ibv_close_device(verbs); + ERROR(errp, "Could not query initial IB port"); + return -EINVAL; + } + + if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { + ib_found = true; + } else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { + roce_found = true; + } + + ibv_close_device(verbs); + + } + + if (roce_found) { + if (ib_found) { + fprintf(stderr, "WARN: migrations may fail:" + " IPv6 over RoCE / iWARP in linux" + " is broken. But since you appear to have a" + " mixed RoCE / IB environment, be sure to only" + " migrate over the IB fabric until the kernel " + " fixes the bug.\n"); + } else { + ERROR(errp, "You only have RoCE / iWARP devices in your systems" + " and your management software has specified '[::]'" + ", but IPv6 over RoCE / iWARP is not supported in Linux."); + return -ENONET; + } + } + + return 0; + } + + /* + * If we have a verbs context, that means that some other than '[::]' was + * used by the management software for binding. In which case we can + * actually warn the user about a potentially broken kernel. + */ + + /* IB ports start with 1, not 0 */ + if (ibv_query_port(verbs, 1, &port_attr)) { + ERROR(errp, "Could not query initial IB port"); + return -EINVAL; + } + + if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) { + ERROR(errp, "Linux kernel's RoCE / iWARP does not support IPv6 " + "(but patches on linux-rdma in progress)"); + return -ENONET; + } + +#endif + + return 0; +} + +/* + * Figure out which RDMA device corresponds to the requested IP hostname + * Also create the initial connection manager identifiers for opening + * the connection. + */ +static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp) +{ + int ret; + struct rdma_addrinfo *res; + char port_str[16]; + struct rdma_cm_event *cm_event; + char ip[40] = "unknown"; + struct rdma_addrinfo *e; + + if (rdma->host == NULL || !strcmp(rdma->host, "")) { + ERROR(errp, "RDMA hostname has not been set"); + return -EINVAL; + } + + /* create CM channel */ + rdma->channel = rdma_create_event_channel(); + if (!rdma->channel) { + ERROR(errp, "could not create CM channel"); + return -EINVAL; + } + + /* create CM id */ + ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP); + if (ret) { + ERROR(errp, "could not create channel id"); + goto err_resolve_create_id; + } + + snprintf(port_str, 16, "%d", rdma->port); + port_str[15] = '\0'; + + ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res); + if (ret < 0) { + ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host); + goto err_resolve_get_addr; + } + + for (e = res; e != NULL; e = e->ai_next) { + inet_ntop(e->ai_family, + &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip); + trace_qemu_rdma_resolve_host_trying(rdma->host, ip); + + ret = rdma_resolve_addr(rdma->cm_id, NULL, e->ai_dst_addr, + RDMA_RESOLVE_TIMEOUT_MS); + if (!ret) { + if (e->ai_family == AF_INET6) { + ret = qemu_rdma_broken_ipv6_kernel(rdma->cm_id->verbs, errp); + if (ret) { + continue; + } + } + goto route; + } + } + + rdma_freeaddrinfo(res); + ERROR(errp, "could not resolve address %s", rdma->host); + goto err_resolve_get_addr; + +route: + rdma_freeaddrinfo(res); + qemu_rdma_dump_gid("source_resolve_addr", rdma->cm_id); + + ret = rdma_get_cm_event(rdma->channel, &cm_event); + if (ret) { + ERROR(errp, "could not perform event_addr_resolved"); + goto err_resolve_get_addr; + } + + if (cm_event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { + ERROR(errp, "result not equal to event_addr_resolved %s", + rdma_event_str(cm_event->event)); + error_report("rdma_resolve_addr"); + rdma_ack_cm_event(cm_event); + ret = -EINVAL; + goto err_resolve_get_addr; + } + rdma_ack_cm_event(cm_event); + + /* resolve route */ + ret = rdma_resolve_route(rdma->cm_id, RDMA_RESOLVE_TIMEOUT_MS); + if (ret) { + ERROR(errp, "could not resolve rdma route"); + goto err_resolve_get_addr; + } + + ret = rdma_get_cm_event(rdma->channel, &cm_event); + if (ret) { + ERROR(errp, "could not perform event_route_resolved"); + goto err_resolve_get_addr; + } + if (cm_event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { + ERROR(errp, "result not equal to event_route_resolved: %s", + rdma_event_str(cm_event->event)); + rdma_ack_cm_event(cm_event); + ret = -EINVAL; + goto err_resolve_get_addr; + } + rdma_ack_cm_event(cm_event); + rdma->verbs = rdma->cm_id->verbs; + qemu_rdma_dump_id("source_resolve_host", rdma->cm_id->verbs); + qemu_rdma_dump_gid("source_resolve_host", rdma->cm_id); + return 0; + +err_resolve_get_addr: + rdma_destroy_id(rdma->cm_id); + rdma->cm_id = NULL; +err_resolve_create_id: + rdma_destroy_event_channel(rdma->channel); + rdma->channel = NULL; + return ret; +} + +/* + * Create protection domain and completion queues + */ +static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma) +{ + /* allocate pd */ + rdma->pd = ibv_alloc_pd(rdma->verbs); + if (!rdma->pd) { + error_report("failed to allocate protection domain"); + return -1; + } + + /* create receive completion channel */ + rdma->recv_comp_channel = ibv_create_comp_channel(rdma->verbs); + if (!rdma->recv_comp_channel) { + error_report("failed to allocate receive completion channel"); + goto err_alloc_pd_cq; + } + + /* + * Completion queue can be filled by read work requests. + */ + rdma->recv_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3), + NULL, rdma->recv_comp_channel, 0); + if (!rdma->recv_cq) { + error_report("failed to allocate receive completion queue"); + goto err_alloc_pd_cq; + } + + /* create send completion channel */ + rdma->send_comp_channel = ibv_create_comp_channel(rdma->verbs); + if (!rdma->send_comp_channel) { + error_report("failed to allocate send completion channel"); + goto err_alloc_pd_cq; + } + + rdma->send_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3), + NULL, rdma->send_comp_channel, 0); + if (!rdma->send_cq) { + error_report("failed to allocate send completion queue"); + goto err_alloc_pd_cq; + } + + return 0; + +err_alloc_pd_cq: + if (rdma->pd) { + ibv_dealloc_pd(rdma->pd); + } + if (rdma->recv_comp_channel) { + ibv_destroy_comp_channel(rdma->recv_comp_channel); + } + if (rdma->send_comp_channel) { + ibv_destroy_comp_channel(rdma->send_comp_channel); + } + if (rdma->recv_cq) { + ibv_destroy_cq(rdma->recv_cq); + rdma->recv_cq = NULL; + } + rdma->pd = NULL; + rdma->recv_comp_channel = NULL; + rdma->send_comp_channel = NULL; + return -1; + +} + +/* + * Create queue pairs. + */ +static int qemu_rdma_alloc_qp(RDMAContext *rdma) +{ + struct ibv_qp_init_attr attr = { 0 }; + int ret; + + attr.cap.max_send_wr = RDMA_SIGNALED_SEND_MAX; + attr.cap.max_recv_wr = 3; + attr.cap.max_send_sge = 1; + attr.cap.max_recv_sge = 1; + attr.send_cq = rdma->send_cq; + attr.recv_cq = rdma->recv_cq; + attr.qp_type = IBV_QPT_RC; + + ret = rdma_create_qp(rdma->cm_id, rdma->pd, &attr); + if (ret) { + return -1; + } + + rdma->qp = rdma->cm_id->qp; + return 0; +} + +/* Check whether On-Demand Paging is supported by RDAM device */ +static bool rdma_support_odp(struct ibv_context *dev) +{ + struct ibv_device_attr_ex attr = {0}; + int ret = ibv_query_device_ex(dev, NULL, &attr); + if (ret) { + return false; + } + + if (attr.odp_caps.general_caps & IBV_ODP_SUPPORT) { + return true; + } + + return false; +} + +/* + * ibv_advise_mr to avoid RNR NAK error as far as possible. + * The responder mr registering with ODP will sent RNR NAK back to + * the requester in the face of the page fault. + */ +static void qemu_rdma_advise_prefetch_mr(struct ibv_pd *pd, uint64_t addr, + uint32_t len, uint32_t lkey, + const char *name, bool wr) +{ +#ifdef HAVE_IBV_ADVISE_MR + int ret; + int advice = wr ? IBV_ADVISE_MR_ADVICE_PREFETCH_WRITE : + IBV_ADVISE_MR_ADVICE_PREFETCH; + struct ibv_sge sg_list = {.lkey = lkey, .addr = addr, .length = len}; + + ret = ibv_advise_mr(pd, advice, + IBV_ADVISE_MR_FLAG_FLUSH, &sg_list, 1); + /* ignore the error */ + if (ret) { + trace_qemu_rdma_advise_mr(name, len, addr, strerror(errno)); + } else { + trace_qemu_rdma_advise_mr(name, len, addr, "successed"); + } +#endif +} + +static int qemu_rdma_reg_whole_ram_blocks(RDMAContext *rdma) +{ + int i; + RDMALocalBlocks *local = &rdma->local_ram_blocks; + + for (i = 0; i < local->nb_blocks; i++) { + int access = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE; + + local->block[i].mr = + ibv_reg_mr(rdma->pd, + local->block[i].local_host_addr, + local->block[i].length, access + ); + + if (!local->block[i].mr && + errno == ENOTSUP && rdma_support_odp(rdma->verbs)) { + access |= IBV_ACCESS_ON_DEMAND; + /* register ODP mr */ + local->block[i].mr = + ibv_reg_mr(rdma->pd, + local->block[i].local_host_addr, + local->block[i].length, access); + trace_qemu_rdma_register_odp_mr(local->block[i].block_name); + + if (local->block[i].mr) { + qemu_rdma_advise_prefetch_mr(rdma->pd, + (uintptr_t)local->block[i].local_host_addr, + local->block[i].length, + local->block[i].mr->lkey, + local->block[i].block_name, + true); + } + } + + if (!local->block[i].mr) { + perror("Failed to register local dest ram block!"); + break; + } + rdma->total_registrations++; + } + + if (i >= local->nb_blocks) { + return 0; + } + + for (i--; i >= 0; i--) { + ibv_dereg_mr(local->block[i].mr); + local->block[i].mr = NULL; + rdma->total_registrations--; + } + + return -1; + +} + +/* + * Find the ram block that corresponds to the page requested to be + * transmitted by QEMU. + * + * Once the block is found, also identify which 'chunk' within that + * block that the page belongs to. + * + * This search cannot fail or the migration will fail. + */ +static int qemu_rdma_search_ram_block(RDMAContext *rdma, + uintptr_t block_offset, + uint64_t offset, + uint64_t length, + uint64_t *block_index, + uint64_t *chunk_index) +{ + uint64_t current_addr = block_offset + offset; + RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap, + (void *) block_offset); + assert(block); + assert(current_addr >= block->offset); + assert((current_addr + length) <= (block->offset + block->length)); + + *block_index = block->index; + *chunk_index = ram_chunk_index(block->local_host_addr, + block->local_host_addr + (current_addr - block->offset)); + + return 0; +} + +/* + * Register a chunk with IB. If the chunk was already registered + * previously, then skip. + * + * Also return the keys associated with the registration needed + * to perform the actual RDMA operation. + */ +static int qemu_rdma_register_and_get_keys(RDMAContext *rdma, + RDMALocalBlock *block, uintptr_t host_addr, + uint32_t *lkey, uint32_t *rkey, int chunk, + uint8_t *chunk_start, uint8_t *chunk_end) +{ + if (block->mr) { + if (lkey) { + *lkey = block->mr->lkey; + } + if (rkey) { + *rkey = block->mr->rkey; + } + return 0; + } + + /* allocate memory to store chunk MRs */ + if (!block->pmr) { + block->pmr = g_new0(struct ibv_mr *, block->nb_chunks); + } + + /* + * If 'rkey', then we're the destination, so grant access to the source. + * + * If 'lkey', then we're the source VM, so grant access only to ourselves. + */ + if (!block->pmr[chunk]) { + uint64_t len = chunk_end - chunk_start; + int access = rkey ? IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE : + 0; + + trace_qemu_rdma_register_and_get_keys(len, chunk_start); + + block->pmr[chunk] = ibv_reg_mr(rdma->pd, chunk_start, len, access); + if (!block->pmr[chunk] && + errno == ENOTSUP && rdma_support_odp(rdma->verbs)) { + access |= IBV_ACCESS_ON_DEMAND; + /* register ODP mr */ + block->pmr[chunk] = ibv_reg_mr(rdma->pd, chunk_start, len, access); + trace_qemu_rdma_register_odp_mr(block->block_name); + + if (block->pmr[chunk]) { + qemu_rdma_advise_prefetch_mr(rdma->pd, (uintptr_t)chunk_start, + len, block->pmr[chunk]->lkey, + block->block_name, rkey); + + } + } + } + if (!block->pmr[chunk]) { + perror("Failed to register chunk!"); + fprintf(stderr, "Chunk details: block: %d chunk index %d" + " start %" PRIuPTR " end %" PRIuPTR + " host %" PRIuPTR + " local %" PRIuPTR " registrations: %d\n", + block->index, chunk, (uintptr_t)chunk_start, + (uintptr_t)chunk_end, host_addr, + (uintptr_t)block->local_host_addr, + rdma->total_registrations); + return -1; + } + rdma->total_registrations++; + + if (lkey) { + *lkey = block->pmr[chunk]->lkey; + } + if (rkey) { + *rkey = block->pmr[chunk]->rkey; + } + return 0; +} + +/* + * Register (at connection time) the memory used for control + * channel messages. + */ +static int qemu_rdma_reg_control(RDMAContext *rdma, int idx) +{ + rdma->wr_data[idx].control_mr = ibv_reg_mr(rdma->pd, + rdma->wr_data[idx].control, RDMA_CONTROL_MAX_BUFFER, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + if (rdma->wr_data[idx].control_mr) { + rdma->total_registrations++; + return 0; + } + error_report("qemu_rdma_reg_control failed"); + return -1; +} + +const char *print_wrid(int wrid) +{ + if (wrid >= RDMA_WRID_RECV_CONTROL) { + return wrid_desc[RDMA_WRID_RECV_CONTROL]; + } + return wrid_desc[wrid]; +} + +/* + * RDMA requires memory registration (mlock/pinning), but this is not good for + * overcommitment. + * + * In preparation for the future where LRU information or workload-specific + * writable writable working set memory access behavior is available to QEMU + * it would be nice to have in place the ability to UN-register/UN-pin + * particular memory regions from the RDMA hardware when it is determine that + * those regions of memory will likely not be accessed again in the near future. + * + * While we do not yet have such information right now, the following + * compile-time option allows us to perform a non-optimized version of this + * behavior. + * + * By uncommenting this option, you will cause *all* RDMA transfers to be + * unregistered immediately after the transfer completes on both sides of the + * connection. This has no effect in 'rdma-pin-all' mode, only regular mode. + * + * This will have a terrible impact on migration performance, so until future + * workload information or LRU information is available, do not attempt to use + * this feature except for basic testing. + */ +/* #define RDMA_UNREGISTRATION_EXAMPLE */ + +/* + * Perform a non-optimized memory unregistration after every transfer + * for demonstration purposes, only if pin-all is not requested. + * + * Potential optimizations: + * 1. Start a new thread to run this function continuously + - for bit clearing + - and for receipt of unregister messages + * 2. Use an LRU. + * 3. Use workload hints. + */ +static int qemu_rdma_unregister_waiting(RDMAContext *rdma) +{ + while (rdma->unregistrations[rdma->unregister_current]) { + int ret; + uint64_t wr_id = rdma->unregistrations[rdma->unregister_current]; + uint64_t chunk = + (wr_id & RDMA_WRID_CHUNK_MASK) >> RDMA_WRID_CHUNK_SHIFT; + uint64_t index = + (wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT; + RDMALocalBlock *block = + &(rdma->local_ram_blocks.block[index]); + RDMARegister reg = { .current_index = index }; + RDMAControlHeader resp = { .type = RDMA_CONTROL_UNREGISTER_FINISHED, + }; + RDMAControlHeader head = { .len = sizeof(RDMARegister), + .type = RDMA_CONTROL_UNREGISTER_REQUEST, + .repeat = 1, + }; + + trace_qemu_rdma_unregister_waiting_proc(chunk, + rdma->unregister_current); + + rdma->unregistrations[rdma->unregister_current] = 0; + rdma->unregister_current++; + + if (rdma->unregister_current == RDMA_SIGNALED_SEND_MAX) { + rdma->unregister_current = 0; + } + + + /* + * Unregistration is speculative (because migration is single-threaded + * and we cannot break the protocol's inifinband message ordering). + * Thus, if the memory is currently being used for transmission, + * then abort the attempt to unregister and try again + * later the next time a completion is received for this memory. + */ + clear_bit(chunk, block->unregister_bitmap); + + if (test_bit(chunk, block->transit_bitmap)) { + trace_qemu_rdma_unregister_waiting_inflight(chunk); + continue; + } + + trace_qemu_rdma_unregister_waiting_send(chunk); + + ret = ibv_dereg_mr(block->pmr[chunk]); + block->pmr[chunk] = NULL; + block->remote_keys[chunk] = 0; + + if (ret != 0) { + perror("unregistration chunk failed"); + return -ret; + } + rdma->total_registrations--; + + reg.key.chunk = chunk; + register_to_network(rdma, ®); + ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®, + &resp, NULL, NULL); + if (ret < 0) { + return ret; + } + + trace_qemu_rdma_unregister_waiting_complete(chunk); + } + + return 0; +} + +static uint64_t qemu_rdma_make_wrid(uint64_t wr_id, uint64_t index, + uint64_t chunk) +{ + uint64_t result = wr_id & RDMA_WRID_TYPE_MASK; + + result |= (index << RDMA_WRID_BLOCK_SHIFT); + result |= (chunk << RDMA_WRID_CHUNK_SHIFT); + + return result; +} + +/* + * Set bit for unregistration in the next iteration. + * We cannot transmit right here, but will unpin later. + */ +static void qemu_rdma_signal_unregister(RDMAContext *rdma, uint64_t index, + uint64_t chunk, uint64_t wr_id) +{ + if (rdma->unregistrations[rdma->unregister_next] != 0) { + error_report("rdma migration: queue is full"); + } else { + RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]); + + if (!test_and_set_bit(chunk, block->unregister_bitmap)) { + trace_qemu_rdma_signal_unregister_append(chunk, + rdma->unregister_next); + + rdma->unregistrations[rdma->unregister_next++] = + qemu_rdma_make_wrid(wr_id, index, chunk); + + if (rdma->unregister_next == RDMA_SIGNALED_SEND_MAX) { + rdma->unregister_next = 0; + } + } else { + trace_qemu_rdma_signal_unregister_already(chunk); + } + } +} + +/* + * Consult the connection manager to see a work request + * (of any kind) has completed. + * Return the work request ID that completed. + */ +static uint64_t qemu_rdma_poll(RDMAContext *rdma, struct ibv_cq *cq, + uint64_t *wr_id_out, uint32_t *byte_len) +{ + int ret; + struct ibv_wc wc; + uint64_t wr_id; + + ret = ibv_poll_cq(cq, 1, &wc); + + if (!ret) { + *wr_id_out = RDMA_WRID_NONE; + return 0; + } + + if (ret < 0) { + error_report("ibv_poll_cq return %d", ret); + return ret; + } + + wr_id = wc.wr_id & RDMA_WRID_TYPE_MASK; + + if (wc.status != IBV_WC_SUCCESS) { + fprintf(stderr, "ibv_poll_cq wc.status=%d %s!\n", + wc.status, ibv_wc_status_str(wc.status)); + fprintf(stderr, "ibv_poll_cq wrid=%s!\n", wrid_desc[wr_id]); + + return -1; + } + + if (rdma->control_ready_expected && + (wr_id >= RDMA_WRID_RECV_CONTROL)) { + trace_qemu_rdma_poll_recv(wrid_desc[RDMA_WRID_RECV_CONTROL], + wr_id - RDMA_WRID_RECV_CONTROL, wr_id, rdma->nb_sent); + rdma->control_ready_expected = 0; + } + + if (wr_id == RDMA_WRID_RDMA_WRITE) { + uint64_t chunk = + (wc.wr_id & RDMA_WRID_CHUNK_MASK) >> RDMA_WRID_CHUNK_SHIFT; + uint64_t index = + (wc.wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT; + RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]); + + trace_qemu_rdma_poll_write(print_wrid(wr_id), wr_id, rdma->nb_sent, + index, chunk, block->local_host_addr, + (void *)(uintptr_t)block->remote_host_addr); + + clear_bit(chunk, block->transit_bitmap); + + if (rdma->nb_sent > 0) { + rdma->nb_sent--; + } + + if (!rdma->pin_all) { + /* + * FYI: If one wanted to signal a specific chunk to be unregistered + * using LRU or workload-specific information, this is the function + * you would call to do so. That chunk would then get asynchronously + * unregistered later. + */ +#ifdef RDMA_UNREGISTRATION_EXAMPLE + qemu_rdma_signal_unregister(rdma, index, chunk, wc.wr_id); +#endif + } + } else { + trace_qemu_rdma_poll_other(print_wrid(wr_id), wr_id, rdma->nb_sent); + } + + *wr_id_out = wc.wr_id; + if (byte_len) { + *byte_len = wc.byte_len; + } + + return 0; +} + +/* Wait for activity on the completion channel. + * Returns 0 on success, none-0 on error. + */ +static int qemu_rdma_wait_comp_channel(RDMAContext *rdma, + struct ibv_comp_channel *comp_channel) +{ + struct rdma_cm_event *cm_event; + int ret = -1; + + /* + * Coroutine doesn't start until migration_fd_process_incoming() + * so don't yield unless we know we're running inside of a coroutine. + */ + if (rdma->migration_started_on_destination && + migration_incoming_get_current()->state == MIGRATION_STATUS_ACTIVE) { + yield_until_fd_readable(comp_channel->fd); + } else { + /* This is the source side, we're in a separate thread + * or destination prior to migration_fd_process_incoming() + * after postcopy, the destination also in a separate thread. + * we can't yield; so we have to poll the fd. + * But we need to be able to handle 'cancel' or an error + * without hanging forever. + */ + while (!rdma->error_state && !rdma->received_error) { + GPollFD pfds[2]; + pfds[0].fd = comp_channel->fd; + pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR; + pfds[0].revents = 0; + + pfds[1].fd = rdma->channel->fd; + pfds[1].events = G_IO_IN | G_IO_HUP | G_IO_ERR; + pfds[1].revents = 0; + + /* 0.1s timeout, should be fine for a 'cancel' */ + switch (qemu_poll_ns(pfds, 2, 100 * 1000 * 1000)) { + case 2: + case 1: /* fd active */ + if (pfds[0].revents) { + return 0; + } + + if (pfds[1].revents) { + ret = rdma_get_cm_event(rdma->channel, &cm_event); + if (ret) { + error_report("failed to get cm event while wait " + "completion channel"); + return -EPIPE; + } + + error_report("receive cm event while wait comp channel," + "cm event is %d", cm_event->event); + if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED || + cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) { + rdma_ack_cm_event(cm_event); + return -EPIPE; + } + rdma_ack_cm_event(cm_event); + } + break; + + case 0: /* Timeout, go around again */ + break; + + default: /* Error of some type - + * I don't trust errno from qemu_poll_ns + */ + error_report("%s: poll failed", __func__); + return -EPIPE; + } + + if (migrate_get_current()->state == MIGRATION_STATUS_CANCELLING) { + /* Bail out and let the cancellation happen */ + return -EPIPE; + } + } + } + + if (rdma->received_error) { + return -EPIPE; + } + return rdma->error_state; +} + +static struct ibv_comp_channel *to_channel(RDMAContext *rdma, int wrid) +{ + return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_comp_channel : + rdma->recv_comp_channel; +} + +static struct ibv_cq *to_cq(RDMAContext *rdma, int wrid) +{ + return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_cq : rdma->recv_cq; +} + +/* + * Block until the next work request has completed. + * + * First poll to see if a work request has already completed, + * otherwise block. + * + * If we encounter completed work requests for IDs other than + * the one we're interested in, then that's generally an error. + * + * The only exception is actual RDMA Write completions. These + * completions only need to be recorded, but do not actually + * need further processing. + */ +static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested, + uint32_t *byte_len) +{ + int num_cq_events = 0, ret = 0; + struct ibv_cq *cq; + void *cq_ctx; + uint64_t wr_id = RDMA_WRID_NONE, wr_id_in; + struct ibv_comp_channel *ch = to_channel(rdma, wrid_requested); + struct ibv_cq *poll_cq = to_cq(rdma, wrid_requested); + + if (ibv_req_notify_cq(poll_cq, 0)) { + return -1; + } + /* poll cq first */ + while (wr_id != wrid_requested) { + ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len); + if (ret < 0) { + return ret; + } + + wr_id = wr_id_in & RDMA_WRID_TYPE_MASK; + + if (wr_id == RDMA_WRID_NONE) { + break; + } + if (wr_id != wrid_requested) { + trace_qemu_rdma_block_for_wrid_miss(print_wrid(wrid_requested), + wrid_requested, print_wrid(wr_id), wr_id); + } + } + + if (wr_id == wrid_requested) { + return 0; + } + + while (1) { + ret = qemu_rdma_wait_comp_channel(rdma, ch); + if (ret) { + goto err_block_for_wrid; + } + + ret = ibv_get_cq_event(ch, &cq, &cq_ctx); + if (ret) { + perror("ibv_get_cq_event"); + goto err_block_for_wrid; + } + + num_cq_events++; + + ret = -ibv_req_notify_cq(cq, 0); + if (ret) { + goto err_block_for_wrid; + } + + while (wr_id != wrid_requested) { + ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len); + if (ret < 0) { + goto err_block_for_wrid; + } + + wr_id = wr_id_in & RDMA_WRID_TYPE_MASK; + + if (wr_id == RDMA_WRID_NONE) { + break; + } + if (wr_id != wrid_requested) { + trace_qemu_rdma_block_for_wrid_miss(print_wrid(wrid_requested), + wrid_requested, print_wrid(wr_id), wr_id); + } + } + + if (wr_id == wrid_requested) { + goto success_block_for_wrid; + } + } + +success_block_for_wrid: + if (num_cq_events) { + ibv_ack_cq_events(cq, num_cq_events); + } + return 0; + +err_block_for_wrid: + if (num_cq_events) { + ibv_ack_cq_events(cq, num_cq_events); + } + + rdma->error_state = ret; + return ret; +} + +/* + * Post a SEND message work request for the control channel + * containing some data and block until the post completes. + */ +static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf, + RDMAControlHeader *head) +{ + int ret = 0; + RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL]; + struct ibv_send_wr *bad_wr; + struct ibv_sge sge = { + .addr = (uintptr_t)(wr->control), + .length = head->len + sizeof(RDMAControlHeader), + .lkey = wr->control_mr->lkey, + }; + struct ibv_send_wr send_wr = { + .wr_id = RDMA_WRID_SEND_CONTROL, + .opcode = IBV_WR_SEND, + .send_flags = IBV_SEND_SIGNALED, + .sg_list = &sge, + .num_sge = 1, + }; + + trace_qemu_rdma_post_send_control(control_desc(head->type)); + + /* + * We don't actually need to do a memcpy() in here if we used + * the "sge" properly, but since we're only sending control messages + * (not RAM in a performance-critical path), then its OK for now. + * + * The copy makes the RDMAControlHeader simpler to manipulate + * for the time being. + */ + assert(head->len <= RDMA_CONTROL_MAX_BUFFER - sizeof(*head)); + memcpy(wr->control, head, sizeof(RDMAControlHeader)); + control_to_network((void *) wr->control); + + if (buf) { + memcpy(wr->control + sizeof(RDMAControlHeader), buf, head->len); + } + + + ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr); + + if (ret > 0) { + error_report("Failed to use post IB SEND for control"); + return -ret; + } + + ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL); + if (ret < 0) { + error_report("rdma migration: send polling control error"); + } + + return ret; +} + +/* + * Post a RECV work request in anticipation of some future receipt + * of data on the control channel. + */ +static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx) +{ + struct ibv_recv_wr *bad_wr; + struct ibv_sge sge = { + .addr = (uintptr_t)(rdma->wr_data[idx].control), + .length = RDMA_CONTROL_MAX_BUFFER, + .lkey = rdma->wr_data[idx].control_mr->lkey, + }; + + struct ibv_recv_wr recv_wr = { + .wr_id = RDMA_WRID_RECV_CONTROL + idx, + .sg_list = &sge, + .num_sge = 1, + }; + + + if (ibv_post_recv(rdma->qp, &recv_wr, &bad_wr)) { + return -1; + } + + return 0; +} + +/* + * Block and wait for a RECV control channel message to arrive. + */ +static int qemu_rdma_exchange_get_response(RDMAContext *rdma, + RDMAControlHeader *head, int expecting, int idx) +{ + uint32_t byte_len; + int ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RECV_CONTROL + idx, + &byte_len); + + if (ret < 0) { + error_report("rdma migration: recv polling control error!"); + return ret; + } + + network_to_control((void *) rdma->wr_data[idx].control); + memcpy(head, rdma->wr_data[idx].control, sizeof(RDMAControlHeader)); + + trace_qemu_rdma_exchange_get_response_start(control_desc(expecting)); + + if (expecting == RDMA_CONTROL_NONE) { + trace_qemu_rdma_exchange_get_response_none(control_desc(head->type), + head->type); + } else if (head->type != expecting || head->type == RDMA_CONTROL_ERROR) { + error_report("Was expecting a %s (%d) control message" + ", but got: %s (%d), length: %d", + control_desc(expecting), expecting, + control_desc(head->type), head->type, head->len); + if (head->type == RDMA_CONTROL_ERROR) { + rdma->received_error = true; + } + return -EIO; + } + if (head->len > RDMA_CONTROL_MAX_BUFFER - sizeof(*head)) { + error_report("too long length: %d", head->len); + return -EINVAL; + } + if (sizeof(*head) + head->len != byte_len) { + error_report("Malformed length: %d byte_len %d", head->len, byte_len); + return -EINVAL; + } + + return 0; +} + +/* + * When a RECV work request has completed, the work request's + * buffer is pointed at the header. + * + * This will advance the pointer to the data portion + * of the control message of the work request's buffer that + * was populated after the work request finished. + */ +static void qemu_rdma_move_header(RDMAContext *rdma, int idx, + RDMAControlHeader *head) +{ + rdma->wr_data[idx].control_len = head->len; + rdma->wr_data[idx].control_curr = + rdma->wr_data[idx].control + sizeof(RDMAControlHeader); +} + +/* + * This is an 'atomic' high-level operation to deliver a single, unified + * control-channel message. + * + * Additionally, if the user is expecting some kind of reply to this message, + * they can request a 'resp' response message be filled in by posting an + * additional work request on behalf of the user and waiting for an additional + * completion. + * + * The extra (optional) response is used during registration to us from having + * to perform an *additional* exchange of message just to provide a response by + * instead piggy-backing on the acknowledgement. + */ +static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head, + uint8_t *data, RDMAControlHeader *resp, + int *resp_idx, + int (*callback)(RDMAContext *rdma)) +{ + int ret = 0; + + /* + * Wait until the dest is ready before attempting to deliver the message + * by waiting for a READY message. + */ + if (rdma->control_ready_expected) { + RDMAControlHeader resp; + ret = qemu_rdma_exchange_get_response(rdma, + &resp, RDMA_CONTROL_READY, RDMA_WRID_READY); + if (ret < 0) { + return ret; + } + } + + /* + * If the user is expecting a response, post a WR in anticipation of it. + */ + if (resp) { + ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA); + if (ret) { + error_report("rdma migration: error posting" + " extra control recv for anticipated result!"); + return ret; + } + } + + /* + * Post a WR to replace the one we just consumed for the READY message. + */ + ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY); + if (ret) { + error_report("rdma migration: error posting first control recv!"); + return ret; + } + + /* + * Deliver the control message that was requested. + */ + ret = qemu_rdma_post_send_control(rdma, data, head); + + if (ret < 0) { + error_report("Failed to send control buffer!"); + return ret; + } + + /* + * If we're expecting a response, block and wait for it. + */ + if (resp) { + if (callback) { + trace_qemu_rdma_exchange_send_issue_callback(); + ret = callback(rdma); + if (ret < 0) { + return ret; + } + } + + trace_qemu_rdma_exchange_send_waiting(control_desc(resp->type)); + ret = qemu_rdma_exchange_get_response(rdma, resp, + resp->type, RDMA_WRID_DATA); + + if (ret < 0) { + return ret; + } + + qemu_rdma_move_header(rdma, RDMA_WRID_DATA, resp); + if (resp_idx) { + *resp_idx = RDMA_WRID_DATA; + } + trace_qemu_rdma_exchange_send_received(control_desc(resp->type)); + } + + rdma->control_ready_expected = 1; + + return 0; +} + +/* + * This is an 'atomic' high-level operation to receive a single, unified + * control-channel message. + */ +static int qemu_rdma_exchange_recv(RDMAContext *rdma, RDMAControlHeader *head, + int expecting) +{ + RDMAControlHeader ready = { + .len = 0, + .type = RDMA_CONTROL_READY, + .repeat = 1, + }; + int ret; + + /* + * Inform the source that we're ready to receive a message. + */ + ret = qemu_rdma_post_send_control(rdma, NULL, &ready); + + if (ret < 0) { + error_report("Failed to send control buffer!"); + return ret; + } + + /* + * Block and wait for the message. + */ + ret = qemu_rdma_exchange_get_response(rdma, head, + expecting, RDMA_WRID_READY); + + if (ret < 0) { + return ret; + } + + qemu_rdma_move_header(rdma, RDMA_WRID_READY, head); + + /* + * Post a new RECV work request to replace the one we just consumed. + */ + ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY); + if (ret) { + error_report("rdma migration: error posting second control recv!"); + return ret; + } + + return 0; +} + +/* + * Write an actual chunk of memory using RDMA. + * + * If we're using dynamic registration on the dest-side, we have to + * send a registration command first. + */ +static int qemu_rdma_write_one(QEMUFile *f, RDMAContext *rdma, + int current_index, uint64_t current_addr, + uint64_t length) +{ + struct ibv_sge sge; + struct ibv_send_wr send_wr = { 0 }; + struct ibv_send_wr *bad_wr; + int reg_result_idx, ret, count = 0; + uint64_t chunk, chunks; + uint8_t *chunk_start, *chunk_end; + RDMALocalBlock *block = &(rdma->local_ram_blocks.block[current_index]); + RDMARegister reg; + RDMARegisterResult *reg_result; + RDMAControlHeader resp = { .type = RDMA_CONTROL_REGISTER_RESULT }; + RDMAControlHeader head = { .len = sizeof(RDMARegister), + .type = RDMA_CONTROL_REGISTER_REQUEST, + .repeat = 1, + }; + +retry: + sge.addr = (uintptr_t)(block->local_host_addr + + (current_addr - block->offset)); + sge.length = length; + + chunk = ram_chunk_index(block->local_host_addr, + (uint8_t *)(uintptr_t)sge.addr); + chunk_start = ram_chunk_start(block, chunk); + + if (block->is_ram_block) { + chunks = length / (1UL << RDMA_REG_CHUNK_SHIFT); + + if (chunks && ((length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) { + chunks--; + } + } else { + chunks = block->length / (1UL << RDMA_REG_CHUNK_SHIFT); + + if (chunks && ((block->length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) { + chunks--; + } + } + + trace_qemu_rdma_write_one_top(chunks + 1, + (chunks + 1) * + (1UL << RDMA_REG_CHUNK_SHIFT) / 1024 / 1024); + + chunk_end = ram_chunk_end(block, chunk + chunks); + + if (!rdma->pin_all) { +#ifdef RDMA_UNREGISTRATION_EXAMPLE + qemu_rdma_unregister_waiting(rdma); +#endif + } + + while (test_bit(chunk, block->transit_bitmap)) { + (void)count; + trace_qemu_rdma_write_one_block(count++, current_index, chunk, + sge.addr, length, rdma->nb_sent, block->nb_chunks); + + ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); + + if (ret < 0) { + error_report("Failed to Wait for previous write to complete " + "block %d chunk %" PRIu64 + " current %" PRIu64 " len %" PRIu64 " %d", + current_index, chunk, sge.addr, length, rdma->nb_sent); + return ret; + } + } + + if (!rdma->pin_all || !block->is_ram_block) { + if (!block->remote_keys[chunk]) { + /* + * This chunk has not yet been registered, so first check to see + * if the entire chunk is zero. If so, tell the other size to + * memset() + madvise() the entire chunk without RDMA. + */ + + if (buffer_is_zero((void *)(uintptr_t)sge.addr, length)) { + RDMACompress comp = { + .offset = current_addr, + .value = 0, + .block_idx = current_index, + .length = length, + }; + + head.len = sizeof(comp); + head.type = RDMA_CONTROL_COMPRESS; + + trace_qemu_rdma_write_one_zero(chunk, sge.length, + current_index, current_addr); + + compress_to_network(rdma, &comp); + ret = qemu_rdma_exchange_send(rdma, &head, + (uint8_t *) &comp, NULL, NULL, NULL); + + if (ret < 0) { + return -EIO; + } + + acct_update_position(f, sge.length, true); + + return 1; + } + + /* + * Otherwise, tell other side to register. + */ + reg.current_index = current_index; + if (block->is_ram_block) { + reg.key.current_addr = current_addr; + } else { + reg.key.chunk = chunk; + } + reg.chunks = chunks; + + trace_qemu_rdma_write_one_sendreg(chunk, sge.length, current_index, + current_addr); + + register_to_network(rdma, ®); + ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®, + &resp, ®_result_idx, NULL); + if (ret < 0) { + return ret; + } + + /* try to overlap this single registration with the one we sent. */ + if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr, + &sge.lkey, NULL, chunk, + chunk_start, chunk_end)) { + error_report("cannot get lkey"); + return -EINVAL; + } + + reg_result = (RDMARegisterResult *) + rdma->wr_data[reg_result_idx].control_curr; + + network_to_result(reg_result); + + trace_qemu_rdma_write_one_recvregres(block->remote_keys[chunk], + reg_result->rkey, chunk); + + block->remote_keys[chunk] = reg_result->rkey; + block->remote_host_addr = reg_result->host_addr; + } else { + /* already registered before */ + if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr, + &sge.lkey, NULL, chunk, + chunk_start, chunk_end)) { + error_report("cannot get lkey!"); + return -EINVAL; + } + } + + send_wr.wr.rdma.rkey = block->remote_keys[chunk]; + } else { + send_wr.wr.rdma.rkey = block->remote_rkey; + + if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr, + &sge.lkey, NULL, chunk, + chunk_start, chunk_end)) { + error_report("cannot get lkey!"); + return -EINVAL; + } + } + + /* + * Encode the ram block index and chunk within this wrid. + * We will use this information at the time of completion + * to figure out which bitmap to check against and then which + * chunk in the bitmap to look for. + */ + send_wr.wr_id = qemu_rdma_make_wrid(RDMA_WRID_RDMA_WRITE, + current_index, chunk); + + send_wr.opcode = IBV_WR_RDMA_WRITE; + send_wr.send_flags = IBV_SEND_SIGNALED; + send_wr.sg_list = &sge; + send_wr.num_sge = 1; + send_wr.wr.rdma.remote_addr = block->remote_host_addr + + (current_addr - block->offset); + + trace_qemu_rdma_write_one_post(chunk, sge.addr, send_wr.wr.rdma.remote_addr, + sge.length); + + /* + * ibv_post_send() does not return negative error numbers, + * per the specification they are positive - no idea why. + */ + ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr); + + if (ret == ENOMEM) { + trace_qemu_rdma_write_one_queue_full(); + ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); + if (ret < 0) { + error_report("rdma migration: failed to make " + "room in full send queue! %d", ret); + return ret; + } + + goto retry; + + } else if (ret > 0) { + perror("rdma migration: post rdma write failed"); + return -ret; + } + + set_bit(chunk, block->transit_bitmap); + acct_update_position(f, sge.length, false); + rdma->total_writes++; + + return 0; +} + +/* + * Push out any unwritten RDMA operations. + * + * We support sending out multiple chunks at the same time. + * Not all of them need to get signaled in the completion queue. + */ +static int qemu_rdma_write_flush(QEMUFile *f, RDMAContext *rdma) +{ + int ret; + + if (!rdma->current_length) { + return 0; + } + + ret = qemu_rdma_write_one(f, rdma, + rdma->current_index, rdma->current_addr, rdma->current_length); + + if (ret < 0) { + return ret; + } + + if (ret == 0) { + rdma->nb_sent++; + trace_qemu_rdma_write_flush(rdma->nb_sent); + } + + rdma->current_length = 0; + rdma->current_addr = 0; + + return 0; +} + +static inline int qemu_rdma_buffer_mergable(RDMAContext *rdma, + uint64_t offset, uint64_t len) +{ + RDMALocalBlock *block; + uint8_t *host_addr; + uint8_t *chunk_end; + + if (rdma->current_index < 0) { + return 0; + } + + if (rdma->current_chunk < 0) { + return 0; + } + + block = &(rdma->local_ram_blocks.block[rdma->current_index]); + host_addr = block->local_host_addr + (offset - block->offset); + chunk_end = ram_chunk_end(block, rdma->current_chunk); + + if (rdma->current_length == 0) { + return 0; + } + + /* + * Only merge into chunk sequentially. + */ + if (offset != (rdma->current_addr + rdma->current_length)) { + return 0; + } + + if (offset < block->offset) { + return 0; + } + + if ((offset + len) > (block->offset + block->length)) { + return 0; + } + + if ((host_addr + len) > chunk_end) { + return 0; + } + + return 1; +} + +/* + * We're not actually writing here, but doing three things: + * + * 1. Identify the chunk the buffer belongs to. + * 2. If the chunk is full or the buffer doesn't belong to the current + * chunk, then start a new chunk and flush() the old chunk. + * 3. To keep the hardware busy, we also group chunks into batches + * and only require that a batch gets acknowledged in the completion + * queue instead of each individual chunk. + */ +static int qemu_rdma_write(QEMUFile *f, RDMAContext *rdma, + uint64_t block_offset, uint64_t offset, + uint64_t len) +{ + uint64_t current_addr = block_offset + offset; + uint64_t index = rdma->current_index; + uint64_t chunk = rdma->current_chunk; + int ret; + + /* If we cannot merge it, we flush the current buffer first. */ + if (!qemu_rdma_buffer_mergable(rdma, current_addr, len)) { + ret = qemu_rdma_write_flush(f, rdma); + if (ret) { + return ret; + } + rdma->current_length = 0; + rdma->current_addr = current_addr; + + ret = qemu_rdma_search_ram_block(rdma, block_offset, + offset, len, &index, &chunk); + if (ret) { + error_report("ram block search failed"); + return ret; + } + rdma->current_index = index; + rdma->current_chunk = chunk; + } + + /* merge it */ + rdma->current_length += len; + + /* flush it if buffer is too large */ + if (rdma->current_length >= RDMA_MERGE_MAX) { + return qemu_rdma_write_flush(f, rdma); + } + + return 0; +} + +static void qemu_rdma_cleanup(RDMAContext *rdma) +{ + int idx; + + if (rdma->cm_id && rdma->connected) { + if ((rdma->error_state || + migrate_get_current()->state == MIGRATION_STATUS_CANCELLING) && + !rdma->received_error) { + RDMAControlHeader head = { .len = 0, + .type = RDMA_CONTROL_ERROR, + .repeat = 1, + }; + error_report("Early error. Sending error."); + qemu_rdma_post_send_control(rdma, NULL, &head); + } + + rdma_disconnect(rdma->cm_id); + trace_qemu_rdma_cleanup_disconnect(); + rdma->connected = false; + } + + if (rdma->channel) { + qemu_set_fd_handler(rdma->channel->fd, NULL, NULL, NULL); + } + g_free(rdma->dest_blocks); + rdma->dest_blocks = NULL; + + for (idx = 0; idx < RDMA_WRID_MAX; idx++) { + if (rdma->wr_data[idx].control_mr) { + rdma->total_registrations--; + ibv_dereg_mr(rdma->wr_data[idx].control_mr); + } + rdma->wr_data[idx].control_mr = NULL; + } + + if (rdma->local_ram_blocks.block) { + while (rdma->local_ram_blocks.nb_blocks) { + rdma_delete_block(rdma, &rdma->local_ram_blocks.block[0]); + } + } + + if (rdma->qp) { + rdma_destroy_qp(rdma->cm_id); + rdma->qp = NULL; + } + if (rdma->recv_cq) { + ibv_destroy_cq(rdma->recv_cq); + rdma->recv_cq = NULL; + } + if (rdma->send_cq) { + ibv_destroy_cq(rdma->send_cq); + rdma->send_cq = NULL; + } + if (rdma->recv_comp_channel) { + ibv_destroy_comp_channel(rdma->recv_comp_channel); + rdma->recv_comp_channel = NULL; + } + if (rdma->send_comp_channel) { + ibv_destroy_comp_channel(rdma->send_comp_channel); + rdma->send_comp_channel = NULL; + } + if (rdma->pd) { + ibv_dealloc_pd(rdma->pd); + rdma->pd = NULL; + } + if (rdma->cm_id) { + rdma_destroy_id(rdma->cm_id); + rdma->cm_id = NULL; + } + + /* the destination side, listen_id and channel is shared */ + if (rdma->listen_id) { + if (!rdma->is_return_path) { + rdma_destroy_id(rdma->listen_id); + } + rdma->listen_id = NULL; + + if (rdma->channel) { + if (!rdma->is_return_path) { + rdma_destroy_event_channel(rdma->channel); + } + rdma->channel = NULL; + } + } + + if (rdma->channel) { + rdma_destroy_event_channel(rdma->channel); + rdma->channel = NULL; + } + g_free(rdma->host); + g_free(rdma->host_port); + rdma->host = NULL; + rdma->host_port = NULL; +} + + +static int qemu_rdma_source_init(RDMAContext *rdma, bool pin_all, Error **errp) +{ + int ret, idx; + Error *local_err = NULL, **temp = &local_err; + + /* + * Will be validated against destination's actual capabilities + * after the connect() completes. + */ + rdma->pin_all = pin_all; + + ret = qemu_rdma_resolve_host(rdma, temp); + if (ret) { + goto err_rdma_source_init; + } + + ret = qemu_rdma_alloc_pd_cq(rdma); + if (ret) { + ERROR(temp, "rdma migration: error allocating pd and cq! Your mlock()" + " limits may be too low. Please check $ ulimit -a # and " + "search for 'ulimit -l' in the output"); + goto err_rdma_source_init; + } + + ret = qemu_rdma_alloc_qp(rdma); + if (ret) { + ERROR(temp, "rdma migration: error allocating qp!"); + goto err_rdma_source_init; + } + + ret = qemu_rdma_init_ram_blocks(rdma); + if (ret) { + ERROR(temp, "rdma migration: error initializing ram blocks!"); + goto err_rdma_source_init; + } + + /* Build the hash that maps from offset to RAMBlock */ + rdma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal); + for (idx = 0; idx < rdma->local_ram_blocks.nb_blocks; idx++) { + g_hash_table_insert(rdma->blockmap, + (void *)(uintptr_t)rdma->local_ram_blocks.block[idx].offset, + &rdma->local_ram_blocks.block[idx]); + } + + for (idx = 0; idx < RDMA_WRID_MAX; idx++) { + ret = qemu_rdma_reg_control(rdma, idx); + if (ret) { + ERROR(temp, "rdma migration: error registering %d control!", + idx); + goto err_rdma_source_init; + } + } + + return 0; + +err_rdma_source_init: + error_propagate(errp, local_err); + qemu_rdma_cleanup(rdma); + return -1; +} + +static int qemu_get_cm_event_timeout(RDMAContext *rdma, + struct rdma_cm_event **cm_event, + long msec, Error **errp) +{ + int ret; + struct pollfd poll_fd = { + .fd = rdma->channel->fd, + .events = POLLIN, + .revents = 0 + }; + + do { + ret = poll(&poll_fd, 1, msec); + } while (ret < 0 && errno == EINTR); + + if (ret == 0) { + ERROR(errp, "poll cm event timeout"); + return -1; + } else if (ret < 0) { + ERROR(errp, "failed to poll cm event, errno=%i", errno); + return -1; + } else if (poll_fd.revents & POLLIN) { + return rdma_get_cm_event(rdma->channel, cm_event); + } else { + ERROR(errp, "no POLLIN event, revent=%x", poll_fd.revents); + return -1; + } +} + +static int qemu_rdma_connect(RDMAContext *rdma, Error **errp, bool return_path) +{ + RDMACapabilities cap = { + .version = RDMA_CONTROL_VERSION_CURRENT, + .flags = 0, + }; + struct rdma_conn_param conn_param = { .initiator_depth = 2, + .retry_count = 5, + .private_data = &cap, + .private_data_len = sizeof(cap), + }; + struct rdma_cm_event *cm_event; + int ret; + + /* + * Only negotiate the capability with destination if the user + * on the source first requested the capability. + */ + if (rdma->pin_all) { + trace_qemu_rdma_connect_pin_all_requested(); + cap.flags |= RDMA_CAPABILITY_PIN_ALL; + } + + caps_to_network(&cap); + + ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY); + if (ret) { + ERROR(errp, "posting second control recv"); + goto err_rdma_source_connect; + } + + ret = rdma_connect(rdma->cm_id, &conn_param); + if (ret) { + perror("rdma_connect"); + ERROR(errp, "connecting to destination!"); + goto err_rdma_source_connect; + } + + if (return_path) { + ret = qemu_get_cm_event_timeout(rdma, &cm_event, 5000, errp); + } else { + ret = rdma_get_cm_event(rdma->channel, &cm_event); + } + if (ret) { + perror("rdma_get_cm_event after rdma_connect"); + ERROR(errp, "connecting to destination!"); + goto err_rdma_source_connect; + } + + if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) { + error_report("rdma_get_cm_event != EVENT_ESTABLISHED after rdma_connect"); + ERROR(errp, "connecting to destination!"); + rdma_ack_cm_event(cm_event); + goto err_rdma_source_connect; + } + rdma->connected = true; + + memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap)); + network_to_caps(&cap); + + /* + * Verify that the *requested* capabilities are supported by the destination + * and disable them otherwise. + */ + if (rdma->pin_all && !(cap.flags & RDMA_CAPABILITY_PIN_ALL)) { + ERROR(errp, "Server cannot support pinning all memory. " + "Will register memory dynamically."); + rdma->pin_all = false; + } + + trace_qemu_rdma_connect_pin_all_outcome(rdma->pin_all); + + rdma_ack_cm_event(cm_event); + + rdma->control_ready_expected = 1; + rdma->nb_sent = 0; + return 0; + +err_rdma_source_connect: + qemu_rdma_cleanup(rdma); + return -1; +} + +static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp) +{ + int ret, idx; + struct rdma_cm_id *listen_id; + char ip[40] = "unknown"; + struct rdma_addrinfo *res, *e; + char port_str[16]; + + for (idx = 0; idx < RDMA_WRID_MAX; idx++) { + rdma->wr_data[idx].control_len = 0; + rdma->wr_data[idx].control_curr = NULL; + } + + if (!rdma->host || !rdma->host[0]) { + ERROR(errp, "RDMA host is not set!"); + rdma->error_state = -EINVAL; + return -1; + } + /* create CM channel */ + rdma->channel = rdma_create_event_channel(); + if (!rdma->channel) { + ERROR(errp, "could not create rdma event channel"); + rdma->error_state = -EINVAL; + return -1; + } + + /* create CM id */ + ret = rdma_create_id(rdma->channel, &listen_id, NULL, RDMA_PS_TCP); + if (ret) { + ERROR(errp, "could not create cm_id!"); + goto err_dest_init_create_listen_id; + } + + snprintf(port_str, 16, "%d", rdma->port); + port_str[15] = '\0'; + + ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res); + if (ret < 0) { + ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host); + goto err_dest_init_bind_addr; + } + + for (e = res; e != NULL; e = e->ai_next) { + inet_ntop(e->ai_family, + &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip); + trace_qemu_rdma_dest_init_trying(rdma->host, ip); + ret = rdma_bind_addr(listen_id, e->ai_dst_addr); + if (ret) { + continue; + } + if (e->ai_family == AF_INET6) { + ret = qemu_rdma_broken_ipv6_kernel(listen_id->verbs, errp); + if (ret) { + continue; + } + } + break; + } + + rdma_freeaddrinfo(res); + if (!e) { + ERROR(errp, "Error: could not rdma_bind_addr!"); + goto err_dest_init_bind_addr; + } + + rdma->listen_id = listen_id; + qemu_rdma_dump_gid("dest_init", listen_id); + return 0; + +err_dest_init_bind_addr: + rdma_destroy_id(listen_id); +err_dest_init_create_listen_id: + rdma_destroy_event_channel(rdma->channel); + rdma->channel = NULL; + rdma->error_state = ret; + return ret; + +} + +static void qemu_rdma_return_path_dest_init(RDMAContext *rdma_return_path, + RDMAContext *rdma) +{ + int idx; + + for (idx = 0; idx < RDMA_WRID_MAX; idx++) { + rdma_return_path->wr_data[idx].control_len = 0; + rdma_return_path->wr_data[idx].control_curr = NULL; + } + + /*the CM channel and CM id is shared*/ + rdma_return_path->channel = rdma->channel; + rdma_return_path->listen_id = rdma->listen_id; + + rdma->return_path = rdma_return_path; + rdma_return_path->return_path = rdma; + rdma_return_path->is_return_path = true; +} + +static void *qemu_rdma_data_init(const char *host_port, Error **errp) +{ + RDMAContext *rdma = NULL; + InetSocketAddress *addr; + + if (host_port) { + rdma = g_new0(RDMAContext, 1); + rdma->current_index = -1; + rdma->current_chunk = -1; + + addr = g_new(InetSocketAddress, 1); + if (!inet_parse(addr, host_port, NULL)) { + rdma->port = atoi(addr->port); + rdma->host = g_strdup(addr->host); + rdma->host_port = g_strdup(host_port); + } else { + ERROR(errp, "bad RDMA migration address '%s'", host_port); + g_free(rdma); + rdma = NULL; + } + + qapi_free_InetSocketAddress(addr); + } + + return rdma; +} + +/* + * QEMUFile interface to the control channel. + * SEND messages for control only. + * VM's ram is handled with regular RDMA messages. + */ +static ssize_t qio_channel_rdma_writev(QIOChannel *ioc, + const struct iovec *iov, + size_t niov, + int *fds, + size_t nfds, + Error **errp) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + QEMUFile *f = rioc->file; + RDMAContext *rdma; + int ret; + ssize_t done = 0; + size_t i; + size_t len = 0; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmaout); + + if (!rdma) { + return -EIO; + } + + CHECK_ERROR_STATE(); + + /* + * Push out any writes that + * we're queued up for VM's ram. + */ + ret = qemu_rdma_write_flush(f, rdma); + if (ret < 0) { + rdma->error_state = ret; + return ret; + } + + for (i = 0; i < niov; i++) { + size_t remaining = iov[i].iov_len; + uint8_t * data = (void *)iov[i].iov_base; + while (remaining) { + RDMAControlHeader head; + + len = MIN(remaining, RDMA_SEND_INCREMENT); + remaining -= len; + + head.len = len; + head.type = RDMA_CONTROL_QEMU_FILE; + + ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL); + + if (ret < 0) { + rdma->error_state = ret; + return ret; + } + + data += len; + done += len; + } + } + + return done; +} + +static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf, + size_t size, int idx) +{ + size_t len = 0; + + if (rdma->wr_data[idx].control_len) { + trace_qemu_rdma_fill(rdma->wr_data[idx].control_len, size); + + len = MIN(size, rdma->wr_data[idx].control_len); + memcpy(buf, rdma->wr_data[idx].control_curr, len); + rdma->wr_data[idx].control_curr += len; + rdma->wr_data[idx].control_len -= len; + } + + return len; +} + +/* + * QEMUFile interface to the control channel. + * RDMA links don't use bytestreams, so we have to + * return bytes to QEMUFile opportunistically. + */ +static ssize_t qio_channel_rdma_readv(QIOChannel *ioc, + const struct iovec *iov, + size_t niov, + int **fds, + size_t *nfds, + Error **errp) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + RDMAContext *rdma; + RDMAControlHeader head; + int ret = 0; + ssize_t i; + size_t done = 0; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmain); + + if (!rdma) { + return -EIO; + } + + CHECK_ERROR_STATE(); + + for (i = 0; i < niov; i++) { + size_t want = iov[i].iov_len; + uint8_t *data = (void *)iov[i].iov_base; + + /* + * First, we hold on to the last SEND message we + * were given and dish out the bytes until we run + * out of bytes. + */ + ret = qemu_rdma_fill(rdma, data, want, 0); + done += ret; + want -= ret; + /* Got what we needed, so go to next iovec */ + if (want == 0) { + continue; + } + + /* If we got any data so far, then don't wait + * for more, just return what we have */ + if (done > 0) { + break; + } + + + /* We've got nothing at all, so lets wait for + * more to arrive + */ + ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE); + + if (ret < 0) { + rdma->error_state = ret; + return ret; + } + + /* + * SEND was received with new bytes, now try again. + */ + ret = qemu_rdma_fill(rdma, data, want, 0); + done += ret; + want -= ret; + + /* Still didn't get enough, so lets just return */ + if (want) { + if (done == 0) { + return QIO_CHANNEL_ERR_BLOCK; + } else { + break; + } + } + } + return done; +} + +/* + * Block until all the outstanding chunks have been delivered by the hardware. + */ +static int qemu_rdma_drain_cq(QEMUFile *f, RDMAContext *rdma) +{ + int ret; + + if (qemu_rdma_write_flush(f, rdma) < 0) { + return -EIO; + } + + while (rdma->nb_sent) { + ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL); + if (ret < 0) { + error_report("rdma migration: complete polling error!"); + return -EIO; + } + } + + qemu_rdma_unregister_waiting(rdma); + + return 0; +} + + +static int qio_channel_rdma_set_blocking(QIOChannel *ioc, + bool blocking, + Error **errp) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + /* XXX we should make readv/writev actually honour this :-) */ + rioc->blocking = blocking; + return 0; +} + + +typedef struct QIOChannelRDMASource QIOChannelRDMASource; +struct QIOChannelRDMASource { + GSource parent; + QIOChannelRDMA *rioc; + GIOCondition condition; +}; + +static gboolean +qio_channel_rdma_source_prepare(GSource *source, + gint *timeout) +{ + QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source; + RDMAContext *rdma; + GIOCondition cond = 0; + *timeout = -1; + + RCU_READ_LOCK_GUARD(); + if (rsource->condition == G_IO_IN) { + rdma = qatomic_rcu_read(&rsource->rioc->rdmain); + } else { + rdma = qatomic_rcu_read(&rsource->rioc->rdmaout); + } + + if (!rdma) { + error_report("RDMAContext is NULL when prepare Gsource"); + return FALSE; + } + + if (rdma->wr_data[0].control_len) { + cond |= G_IO_IN; + } + cond |= G_IO_OUT; + + return cond & rsource->condition; +} + +static gboolean +qio_channel_rdma_source_check(GSource *source) +{ + QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source; + RDMAContext *rdma; + GIOCondition cond = 0; + + RCU_READ_LOCK_GUARD(); + if (rsource->condition == G_IO_IN) { + rdma = qatomic_rcu_read(&rsource->rioc->rdmain); + } else { + rdma = qatomic_rcu_read(&rsource->rioc->rdmaout); + } + + if (!rdma) { + error_report("RDMAContext is NULL when check Gsource"); + return FALSE; + } + + if (rdma->wr_data[0].control_len) { + cond |= G_IO_IN; + } + cond |= G_IO_OUT; + + return cond & rsource->condition; +} + +static gboolean +qio_channel_rdma_source_dispatch(GSource *source, + GSourceFunc callback, + gpointer user_data) +{ + QIOChannelFunc func = (QIOChannelFunc)callback; + QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source; + RDMAContext *rdma; + GIOCondition cond = 0; + + RCU_READ_LOCK_GUARD(); + if (rsource->condition == G_IO_IN) { + rdma = qatomic_rcu_read(&rsource->rioc->rdmain); + } else { + rdma = qatomic_rcu_read(&rsource->rioc->rdmaout); + } + + if (!rdma) { + error_report("RDMAContext is NULL when dispatch Gsource"); + return FALSE; + } + + if (rdma->wr_data[0].control_len) { + cond |= G_IO_IN; + } + cond |= G_IO_OUT; + + return (*func)(QIO_CHANNEL(rsource->rioc), + (cond & rsource->condition), + user_data); +} + +static void +qio_channel_rdma_source_finalize(GSource *source) +{ + QIOChannelRDMASource *ssource = (QIOChannelRDMASource *)source; + + object_unref(OBJECT(ssource->rioc)); +} + +GSourceFuncs qio_channel_rdma_source_funcs = { + qio_channel_rdma_source_prepare, + qio_channel_rdma_source_check, + qio_channel_rdma_source_dispatch, + qio_channel_rdma_source_finalize +}; + +static GSource *qio_channel_rdma_create_watch(QIOChannel *ioc, + GIOCondition condition) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + QIOChannelRDMASource *ssource; + GSource *source; + + source = g_source_new(&qio_channel_rdma_source_funcs, + sizeof(QIOChannelRDMASource)); + ssource = (QIOChannelRDMASource *)source; + + ssource->rioc = rioc; + object_ref(OBJECT(rioc)); + + ssource->condition = condition; + + return source; +} + +static void qio_channel_rdma_set_aio_fd_handler(QIOChannel *ioc, + AioContext *ctx, + IOHandler *io_read, + IOHandler *io_write, + void *opaque) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + if (io_read) { + aio_set_fd_handler(ctx, rioc->rdmain->recv_comp_channel->fd, + false, io_read, io_write, NULL, opaque); + aio_set_fd_handler(ctx, rioc->rdmain->send_comp_channel->fd, + false, io_read, io_write, NULL, opaque); + } else { + aio_set_fd_handler(ctx, rioc->rdmaout->recv_comp_channel->fd, + false, io_read, io_write, NULL, opaque); + aio_set_fd_handler(ctx, rioc->rdmaout->send_comp_channel->fd, + false, io_read, io_write, NULL, opaque); + } +} + +struct rdma_close_rcu { + struct rcu_head rcu; + RDMAContext *rdmain; + RDMAContext *rdmaout; +}; + +/* callback from qio_channel_rdma_close via call_rcu */ +static void qio_channel_rdma_close_rcu(struct rdma_close_rcu *rcu) +{ + if (rcu->rdmain) { + qemu_rdma_cleanup(rcu->rdmain); + } + + if (rcu->rdmaout) { + qemu_rdma_cleanup(rcu->rdmaout); + } + + g_free(rcu->rdmain); + g_free(rcu->rdmaout); + g_free(rcu); +} + +static int qio_channel_rdma_close(QIOChannel *ioc, + Error **errp) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + RDMAContext *rdmain, *rdmaout; + struct rdma_close_rcu *rcu = g_new(struct rdma_close_rcu, 1); + + trace_qemu_rdma_close(); + + rdmain = rioc->rdmain; + if (rdmain) { + qatomic_rcu_set(&rioc->rdmain, NULL); + } + + rdmaout = rioc->rdmaout; + if (rdmaout) { + qatomic_rcu_set(&rioc->rdmaout, NULL); + } + + rcu->rdmain = rdmain; + rcu->rdmaout = rdmaout; + call_rcu(rcu, qio_channel_rdma_close_rcu, rcu); + + return 0; +} + +static int +qio_channel_rdma_shutdown(QIOChannel *ioc, + QIOChannelShutdown how, + Error **errp) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc); + RDMAContext *rdmain, *rdmaout; + + RCU_READ_LOCK_GUARD(); + + rdmain = qatomic_rcu_read(&rioc->rdmain); + rdmaout = qatomic_rcu_read(&rioc->rdmain); + + switch (how) { + case QIO_CHANNEL_SHUTDOWN_READ: + if (rdmain) { + rdmain->error_state = -1; + } + break; + case QIO_CHANNEL_SHUTDOWN_WRITE: + if (rdmaout) { + rdmaout->error_state = -1; + } + break; + case QIO_CHANNEL_SHUTDOWN_BOTH: + default: + if (rdmain) { + rdmain->error_state = -1; + } + if (rdmaout) { + rdmaout->error_state = -1; + } + break; + } + + return 0; +} + +/* + * Parameters: + * @offset == 0 : + * This means that 'block_offset' is a full virtual address that does not + * belong to a RAMBlock of the virtual machine and instead + * represents a private malloc'd memory area that the caller wishes to + * transfer. + * + * @offset != 0 : + * Offset is an offset to be added to block_offset and used + * to also lookup the corresponding RAMBlock. + * + * @size > 0 : + * Initiate an transfer this size. + * + * @size == 0 : + * A 'hint' or 'advice' that means that we wish to speculatively + * and asynchronously unregister this memory. In this case, there is no + * guarantee that the unregister will actually happen, for example, + * if the memory is being actively transmitted. Additionally, the memory + * may be re-registered at any future time if a write within the same + * chunk was requested again, even if you attempted to unregister it + * here. + * + * @size < 0 : TODO, not yet supported + * Unregister the memory NOW. This means that the caller does not + * expect there to be any future RDMA transfers and we just want to clean + * things up. This is used in case the upper layer owns the memory and + * cannot wait for qemu_fclose() to occur. + * + * @bytes_sent : User-specificed pointer to indicate how many bytes were + * sent. Usually, this will not be more than a few bytes of + * the protocol because most transfers are sent asynchronously. + */ +static size_t qemu_rdma_save_page(QEMUFile *f, void *opaque, + ram_addr_t block_offset, ram_addr_t offset, + size_t size, uint64_t *bytes_sent) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque); + RDMAContext *rdma; + int ret; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmaout); + + if (!rdma) { + return -EIO; + } + + CHECK_ERROR_STATE(); + + if (migration_in_postcopy()) { + return RAM_SAVE_CONTROL_NOT_SUPP; + } + + qemu_fflush(f); + + if (size > 0) { + /* + * Add this page to the current 'chunk'. If the chunk + * is full, or the page doesn't belong to the current chunk, + * an actual RDMA write will occur and a new chunk will be formed. + */ + ret = qemu_rdma_write(f, rdma, block_offset, offset, size); + if (ret < 0) { + error_report("rdma migration: write error! %d", ret); + goto err; + } + + /* + * We always return 1 bytes because the RDMA + * protocol is completely asynchronous. We do not yet know + * whether an identified chunk is zero or not because we're + * waiting for other pages to potentially be merged with + * the current chunk. So, we have to call qemu_update_position() + * later on when the actual write occurs. + */ + if (bytes_sent) { + *bytes_sent = 1; + } + } else { + uint64_t index, chunk; + + /* TODO: Change QEMUFileOps prototype to be signed: size_t => long + if (size < 0) { + ret = qemu_rdma_drain_cq(f, rdma); + if (ret < 0) { + fprintf(stderr, "rdma: failed to synchronously drain" + " completion queue before unregistration.\n"); + goto err; + } + } + */ + + ret = qemu_rdma_search_ram_block(rdma, block_offset, + offset, size, &index, &chunk); + + if (ret) { + error_report("ram block search failed"); + goto err; + } + + qemu_rdma_signal_unregister(rdma, index, chunk, 0); + + /* + * TODO: Synchronous, guaranteed unregistration (should not occur during + * fast-path). Otherwise, unregisters will process on the next call to + * qemu_rdma_drain_cq() + if (size < 0) { + qemu_rdma_unregister_waiting(rdma); + } + */ + } + + /* + * Drain the Completion Queue if possible, but do not block, + * just poll. + * + * If nothing to poll, the end of the iteration will do this + * again to make sure we don't overflow the request queue. + */ + while (1) { + uint64_t wr_id, wr_id_in; + int ret = qemu_rdma_poll(rdma, rdma->recv_cq, &wr_id_in, NULL); + if (ret < 0) { + error_report("rdma migration: polling error! %d", ret); + goto err; + } + + wr_id = wr_id_in & RDMA_WRID_TYPE_MASK; + + if (wr_id == RDMA_WRID_NONE) { + break; + } + } + + while (1) { + uint64_t wr_id, wr_id_in; + int ret = qemu_rdma_poll(rdma, rdma->send_cq, &wr_id_in, NULL); + if (ret < 0) { + error_report("rdma migration: polling error! %d", ret); + goto err; + } + + wr_id = wr_id_in & RDMA_WRID_TYPE_MASK; + + if (wr_id == RDMA_WRID_NONE) { + break; + } + } + + return RAM_SAVE_CONTROL_DELAYED; +err: + rdma->error_state = ret; + return ret; +} + +static void rdma_accept_incoming_migration(void *opaque); + +static void rdma_cm_poll_handler(void *opaque) +{ + RDMAContext *rdma = opaque; + int ret; + struct rdma_cm_event *cm_event; + MigrationIncomingState *mis = migration_incoming_get_current(); + + ret = rdma_get_cm_event(rdma->channel, &cm_event); + if (ret) { + error_report("get_cm_event failed %d", errno); + return; + } + + if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED || + cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) { + if (!rdma->error_state && + migration_incoming_get_current()->state != + MIGRATION_STATUS_COMPLETED) { + error_report("receive cm event, cm event is %d", cm_event->event); + rdma->error_state = -EPIPE; + if (rdma->return_path) { + rdma->return_path->error_state = -EPIPE; + } + } + rdma_ack_cm_event(cm_event); + + if (mis->migration_incoming_co) { + qemu_coroutine_enter(mis->migration_incoming_co); + } + return; + } + rdma_ack_cm_event(cm_event); +} + +static int qemu_rdma_accept(RDMAContext *rdma) +{ + RDMACapabilities cap; + struct rdma_conn_param conn_param = { + .responder_resources = 2, + .private_data = &cap, + .private_data_len = sizeof(cap), + }; + RDMAContext *rdma_return_path = NULL; + struct rdma_cm_event *cm_event; + struct ibv_context *verbs; + int ret = -EINVAL; + int idx; + + ret = rdma_get_cm_event(rdma->channel, &cm_event); + if (ret) { + goto err_rdma_dest_wait; + } + + if (cm_event->event != RDMA_CM_EVENT_CONNECT_REQUEST) { + rdma_ack_cm_event(cm_event); + goto err_rdma_dest_wait; + } + + /* + * initialize the RDMAContext for return path for postcopy after first + * connection request reached. + */ + if (migrate_postcopy() && !rdma->is_return_path) { + rdma_return_path = qemu_rdma_data_init(rdma->host_port, NULL); + if (rdma_return_path == NULL) { + rdma_ack_cm_event(cm_event); + goto err_rdma_dest_wait; + } + + qemu_rdma_return_path_dest_init(rdma_return_path, rdma); + } + + memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap)); + + network_to_caps(&cap); + + if (cap.version < 1 || cap.version > RDMA_CONTROL_VERSION_CURRENT) { + error_report("Unknown source RDMA version: %d, bailing...", + cap.version); + rdma_ack_cm_event(cm_event); + goto err_rdma_dest_wait; + } + + /* + * Respond with only the capabilities this version of QEMU knows about. + */ + cap.flags &= known_capabilities; + + /* + * Enable the ones that we do know about. + * Add other checks here as new ones are introduced. + */ + if (cap.flags & RDMA_CAPABILITY_PIN_ALL) { + rdma->pin_all = true; + } + + rdma->cm_id = cm_event->id; + verbs = cm_event->id->verbs; + + rdma_ack_cm_event(cm_event); + + trace_qemu_rdma_accept_pin_state(rdma->pin_all); + + caps_to_network(&cap); + + trace_qemu_rdma_accept_pin_verbsc(verbs); + + if (!rdma->verbs) { + rdma->verbs = verbs; + } else if (rdma->verbs != verbs) { + error_report("ibv context not matching %p, %p!", rdma->verbs, + verbs); + goto err_rdma_dest_wait; + } + + qemu_rdma_dump_id("dest_init", verbs); + + ret = qemu_rdma_alloc_pd_cq(rdma); + if (ret) { + error_report("rdma migration: error allocating pd and cq!"); + goto err_rdma_dest_wait; + } + + ret = qemu_rdma_alloc_qp(rdma); + if (ret) { + error_report("rdma migration: error allocating qp!"); + goto err_rdma_dest_wait; + } + + ret = qemu_rdma_init_ram_blocks(rdma); + if (ret) { + error_report("rdma migration: error initializing ram blocks!"); + goto err_rdma_dest_wait; + } + + for (idx = 0; idx < RDMA_WRID_MAX; idx++) { + ret = qemu_rdma_reg_control(rdma, idx); + if (ret) { + error_report("rdma: error registering %d control", idx); + goto err_rdma_dest_wait; + } + } + + /* Accept the second connection request for return path */ + if (migrate_postcopy() && !rdma->is_return_path) { + qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration, + NULL, + (void *)(intptr_t)rdma->return_path); + } else { + qemu_set_fd_handler(rdma->channel->fd, rdma_cm_poll_handler, + NULL, rdma); + } + + ret = rdma_accept(rdma->cm_id, &conn_param); + if (ret) { + error_report("rdma_accept returns %d", ret); + goto err_rdma_dest_wait; + } + + ret = rdma_get_cm_event(rdma->channel, &cm_event); + if (ret) { + error_report("rdma_accept get_cm_event failed %d", ret); + goto err_rdma_dest_wait; + } + + if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) { + error_report("rdma_accept not event established"); + rdma_ack_cm_event(cm_event); + goto err_rdma_dest_wait; + } + + rdma_ack_cm_event(cm_event); + rdma->connected = true; + + ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY); + if (ret) { + error_report("rdma migration: error posting second control recv"); + goto err_rdma_dest_wait; + } + + qemu_rdma_dump_gid("dest_connect", rdma->cm_id); + + return 0; + +err_rdma_dest_wait: + rdma->error_state = ret; + qemu_rdma_cleanup(rdma); + g_free(rdma_return_path); + return ret; +} + +static int dest_ram_sort_func(const void *a, const void *b) +{ + unsigned int a_index = ((const RDMALocalBlock *)a)->src_index; + unsigned int b_index = ((const RDMALocalBlock *)b)->src_index; + + return (a_index < b_index) ? -1 : (a_index != b_index); +} + +/* + * During each iteration of the migration, we listen for instructions + * by the source VM to perform dynamic page registrations before they + * can perform RDMA operations. + * + * We respond with the 'rkey'. + * + * Keep doing this until the source tells us to stop. + */ +static int qemu_rdma_registration_handle(QEMUFile *f, void *opaque) +{ + RDMAControlHeader reg_resp = { .len = sizeof(RDMARegisterResult), + .type = RDMA_CONTROL_REGISTER_RESULT, + .repeat = 0, + }; + RDMAControlHeader unreg_resp = { .len = 0, + .type = RDMA_CONTROL_UNREGISTER_FINISHED, + .repeat = 0, + }; + RDMAControlHeader blocks = { .type = RDMA_CONTROL_RAM_BLOCKS_RESULT, + .repeat = 1 }; + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque); + RDMAContext *rdma; + RDMALocalBlocks *local; + RDMAControlHeader head; + RDMARegister *reg, *registers; + RDMACompress *comp; + RDMARegisterResult *reg_result; + static RDMARegisterResult results[RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE]; + RDMALocalBlock *block; + void *host_addr; + int ret = 0; + int idx = 0; + int count = 0; + int i = 0; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmain); + + if (!rdma) { + return -EIO; + } + + CHECK_ERROR_STATE(); + + local = &rdma->local_ram_blocks; + do { + trace_qemu_rdma_registration_handle_wait(); + + ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE); + + if (ret < 0) { + break; + } + + if (head.repeat > RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE) { + error_report("rdma: Too many requests in this message (%d)." + "Bailing.", head.repeat); + ret = -EIO; + break; + } + + switch (head.type) { + case RDMA_CONTROL_COMPRESS: + comp = (RDMACompress *) rdma->wr_data[idx].control_curr; + network_to_compress(comp); + + trace_qemu_rdma_registration_handle_compress(comp->length, + comp->block_idx, + comp->offset); + if (comp->block_idx >= rdma->local_ram_blocks.nb_blocks) { + error_report("rdma: 'compress' bad block index %u (vs %d)", + (unsigned int)comp->block_idx, + rdma->local_ram_blocks.nb_blocks); + ret = -EIO; + goto out; + } + block = &(rdma->local_ram_blocks.block[comp->block_idx]); + + host_addr = block->local_host_addr + + (comp->offset - block->offset); + + ram_handle_compressed(host_addr, comp->value, comp->length); + break; + + case RDMA_CONTROL_REGISTER_FINISHED: + trace_qemu_rdma_registration_handle_finished(); + goto out; + + case RDMA_CONTROL_RAM_BLOCKS_REQUEST: + trace_qemu_rdma_registration_handle_ram_blocks(); + + /* Sort our local RAM Block list so it's the same as the source, + * we can do this since we've filled in a src_index in the list + * as we received the RAMBlock list earlier. + */ + qsort(rdma->local_ram_blocks.block, + rdma->local_ram_blocks.nb_blocks, + sizeof(RDMALocalBlock), dest_ram_sort_func); + for (i = 0; i < local->nb_blocks; i++) { + local->block[i].index = i; + } + + if (rdma->pin_all) { + ret = qemu_rdma_reg_whole_ram_blocks(rdma); + if (ret) { + error_report("rdma migration: error dest " + "registering ram blocks"); + goto out; + } + } + + /* + * Dest uses this to prepare to transmit the RAMBlock descriptions + * to the source VM after connection setup. + * Both sides use the "remote" structure to communicate and update + * their "local" descriptions with what was sent. + */ + for (i = 0; i < local->nb_blocks; i++) { + rdma->dest_blocks[i].remote_host_addr = + (uintptr_t)(local->block[i].local_host_addr); + + if (rdma->pin_all) { + rdma->dest_blocks[i].remote_rkey = local->block[i].mr->rkey; + } + + rdma->dest_blocks[i].offset = local->block[i].offset; + rdma->dest_blocks[i].length = local->block[i].length; + + dest_block_to_network(&rdma->dest_blocks[i]); + trace_qemu_rdma_registration_handle_ram_blocks_loop( + local->block[i].block_name, + local->block[i].offset, + local->block[i].length, + local->block[i].local_host_addr, + local->block[i].src_index); + } + + blocks.len = rdma->local_ram_blocks.nb_blocks + * sizeof(RDMADestBlock); + + + ret = qemu_rdma_post_send_control(rdma, + (uint8_t *) rdma->dest_blocks, &blocks); + + if (ret < 0) { + error_report("rdma migration: error sending remote info"); + goto out; + } + + break; + case RDMA_CONTROL_REGISTER_REQUEST: + trace_qemu_rdma_registration_handle_register(head.repeat); + + reg_resp.repeat = head.repeat; + registers = (RDMARegister *) rdma->wr_data[idx].control_curr; + + for (count = 0; count < head.repeat; count++) { + uint64_t chunk; + uint8_t *chunk_start, *chunk_end; + + reg = ®isters[count]; + network_to_register(reg); + + reg_result = &results[count]; + + trace_qemu_rdma_registration_handle_register_loop(count, + reg->current_index, reg->key.current_addr, reg->chunks); + + if (reg->current_index >= rdma->local_ram_blocks.nb_blocks) { + error_report("rdma: 'register' bad block index %u (vs %d)", + (unsigned int)reg->current_index, + rdma->local_ram_blocks.nb_blocks); + ret = -ENOENT; + goto out; + } + block = &(rdma->local_ram_blocks.block[reg->current_index]); + if (block->is_ram_block) { + if (block->offset > reg->key.current_addr) { + error_report("rdma: bad register address for block %s" + " offset: %" PRIx64 " current_addr: %" PRIx64, + block->block_name, block->offset, + reg->key.current_addr); + ret = -ERANGE; + goto out; + } + host_addr = (block->local_host_addr + + (reg->key.current_addr - block->offset)); + chunk = ram_chunk_index(block->local_host_addr, + (uint8_t *) host_addr); + } else { + chunk = reg->key.chunk; + host_addr = block->local_host_addr + + (reg->key.chunk * (1UL << RDMA_REG_CHUNK_SHIFT)); + /* Check for particularly bad chunk value */ + if (host_addr < (void *)block->local_host_addr) { + error_report("rdma: bad chunk for block %s" + " chunk: %" PRIx64, + block->block_name, reg->key.chunk); + ret = -ERANGE; + goto out; + } + } + chunk_start = ram_chunk_start(block, chunk); + chunk_end = ram_chunk_end(block, chunk + reg->chunks); + /* avoid "-Waddress-of-packed-member" warning */ + uint32_t tmp_rkey = 0; + if (qemu_rdma_register_and_get_keys(rdma, block, + (uintptr_t)host_addr, NULL, &tmp_rkey, + chunk, chunk_start, chunk_end)) { + error_report("cannot get rkey"); + ret = -EINVAL; + goto out; + } + reg_result->rkey = tmp_rkey; + + reg_result->host_addr = (uintptr_t)block->local_host_addr; + + trace_qemu_rdma_registration_handle_register_rkey( + reg_result->rkey); + + result_to_network(reg_result); + } + + ret = qemu_rdma_post_send_control(rdma, + (uint8_t *) results, ®_resp); + + if (ret < 0) { + error_report("Failed to send control buffer"); + goto out; + } + break; + case RDMA_CONTROL_UNREGISTER_REQUEST: + trace_qemu_rdma_registration_handle_unregister(head.repeat); + unreg_resp.repeat = head.repeat; + registers = (RDMARegister *) rdma->wr_data[idx].control_curr; + + for (count = 0; count < head.repeat; count++) { + reg = ®isters[count]; + network_to_register(reg); + + trace_qemu_rdma_registration_handle_unregister_loop(count, + reg->current_index, reg->key.chunk); + + block = &(rdma->local_ram_blocks.block[reg->current_index]); + + ret = ibv_dereg_mr(block->pmr[reg->key.chunk]); + block->pmr[reg->key.chunk] = NULL; + + if (ret != 0) { + perror("rdma unregistration chunk failed"); + ret = -ret; + goto out; + } + + rdma->total_registrations--; + + trace_qemu_rdma_registration_handle_unregister_success( + reg->key.chunk); + } + + ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp); + + if (ret < 0) { + error_report("Failed to send control buffer"); + goto out; + } + break; + case RDMA_CONTROL_REGISTER_RESULT: + error_report("Invalid RESULT message at dest."); + ret = -EIO; + goto out; + default: + error_report("Unknown control message %s", control_desc(head.type)); + ret = -EIO; + goto out; + } + } while (1); +out: + if (ret < 0) { + rdma->error_state = ret; + } + return ret; +} + +/* Destination: + * Called via a ram_control_load_hook during the initial RAM load section which + * lists the RAMBlocks by name. This lets us know the order of the RAMBlocks + * on the source. + * We've already built our local RAMBlock list, but not yet sent the list to + * the source. + */ +static int +rdma_block_notification_handle(QIOChannelRDMA *rioc, const char *name) +{ + RDMAContext *rdma; + int curr; + int found = -1; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmain); + + if (!rdma) { + return -EIO; + } + + /* Find the matching RAMBlock in our local list */ + for (curr = 0; curr < rdma->local_ram_blocks.nb_blocks; curr++) { + if (!strcmp(rdma->local_ram_blocks.block[curr].block_name, name)) { + found = curr; + break; + } + } + + if (found == -1) { + error_report("RAMBlock '%s' not found on destination", name); + return -ENOENT; + } + + rdma->local_ram_blocks.block[curr].src_index = rdma->next_src_index; + trace_rdma_block_notification_handle(name, rdma->next_src_index); + rdma->next_src_index++; + + return 0; +} + +static int rdma_load_hook(QEMUFile *f, void *opaque, uint64_t flags, void *data) +{ + switch (flags) { + case RAM_CONTROL_BLOCK_REG: + return rdma_block_notification_handle(opaque, data); + + case RAM_CONTROL_HOOK: + return qemu_rdma_registration_handle(f, opaque); + + default: + /* Shouldn't be called with any other values */ + abort(); + } +} + +static int qemu_rdma_registration_start(QEMUFile *f, void *opaque, + uint64_t flags, void *data) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque); + RDMAContext *rdma; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmaout); + if (!rdma) { + return -EIO; + } + + CHECK_ERROR_STATE(); + + if (migration_in_postcopy()) { + return 0; + } + + trace_qemu_rdma_registration_start(flags); + qemu_put_be64(f, RAM_SAVE_FLAG_HOOK); + qemu_fflush(f); + + return 0; +} + +/* + * Inform dest that dynamic registrations are done for now. + * First, flush writes, if any. + */ +static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque, + uint64_t flags, void *data) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque); + RDMAContext *rdma; + RDMAControlHeader head = { .len = 0, .repeat = 1 }; + int ret = 0; + + RCU_READ_LOCK_GUARD(); + rdma = qatomic_rcu_read(&rioc->rdmaout); + if (!rdma) { + return -EIO; + } + + CHECK_ERROR_STATE(); + + if (migration_in_postcopy()) { + return 0; + } + + qemu_fflush(f); + ret = qemu_rdma_drain_cq(f, rdma); + + if (ret < 0) { + goto err; + } + + if (flags == RAM_CONTROL_SETUP) { + RDMAControlHeader resp = {.type = RDMA_CONTROL_RAM_BLOCKS_RESULT }; + RDMALocalBlocks *local = &rdma->local_ram_blocks; + int reg_result_idx, i, nb_dest_blocks; + + head.type = RDMA_CONTROL_RAM_BLOCKS_REQUEST; + trace_qemu_rdma_registration_stop_ram(); + + /* + * Make sure that we parallelize the pinning on both sides. + * For very large guests, doing this serially takes a really + * long time, so we have to 'interleave' the pinning locally + * with the control messages by performing the pinning on this + * side before we receive the control response from the other + * side that the pinning has completed. + */ + ret = qemu_rdma_exchange_send(rdma, &head, NULL, &resp, + ®_result_idx, rdma->pin_all ? + qemu_rdma_reg_whole_ram_blocks : NULL); + if (ret < 0) { + fprintf(stderr, "receiving remote info!"); + return ret; + } + + nb_dest_blocks = resp.len / sizeof(RDMADestBlock); + + /* + * The protocol uses two different sets of rkeys (mutually exclusive): + * 1. One key to represent the virtual address of the entire ram block. + * (dynamic chunk registration disabled - pin everything with one rkey.) + * 2. One to represent individual chunks within a ram block. + * (dynamic chunk registration enabled - pin individual chunks.) + * + * Once the capability is successfully negotiated, the destination transmits + * the keys to use (or sends them later) including the virtual addresses + * and then propagates the remote ram block descriptions to his local copy. + */ + + if (local->nb_blocks != nb_dest_blocks) { + fprintf(stderr, "ram blocks mismatch (Number of blocks %d vs %d) " + "Your QEMU command line parameters are probably " + "not identical on both the source and destination.", + local->nb_blocks, nb_dest_blocks); + rdma->error_state = -EINVAL; + return -EINVAL; + } + + qemu_rdma_move_header(rdma, reg_result_idx, &resp); + memcpy(rdma->dest_blocks, + rdma->wr_data[reg_result_idx].control_curr, resp.len); + for (i = 0; i < nb_dest_blocks; i++) { + network_to_dest_block(&rdma->dest_blocks[i]); + + /* We require that the blocks are in the same order */ + if (rdma->dest_blocks[i].length != local->block[i].length) { + fprintf(stderr, "Block %s/%d has a different length %" PRIu64 + "vs %" PRIu64, local->block[i].block_name, i, + local->block[i].length, + rdma->dest_blocks[i].length); + rdma->error_state = -EINVAL; + return -EINVAL; + } + local->block[i].remote_host_addr = + rdma->dest_blocks[i].remote_host_addr; + local->block[i].remote_rkey = rdma->dest_blocks[i].remote_rkey; + } + } + + trace_qemu_rdma_registration_stop(flags); + + head.type = RDMA_CONTROL_REGISTER_FINISHED; + ret = qemu_rdma_exchange_send(rdma, &head, NULL, NULL, NULL, NULL); + + if (ret < 0) { + goto err; + } + + return 0; +err: + rdma->error_state = ret; + return ret; +} + +static const QEMUFileHooks rdma_read_hooks = { + .hook_ram_load = rdma_load_hook, +}; + +static const QEMUFileHooks rdma_write_hooks = { + .before_ram_iterate = qemu_rdma_registration_start, + .after_ram_iterate = qemu_rdma_registration_stop, + .save_page = qemu_rdma_save_page, +}; + + +static void qio_channel_rdma_finalize(Object *obj) +{ + QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(obj); + if (rioc->rdmain) { + qemu_rdma_cleanup(rioc->rdmain); + g_free(rioc->rdmain); + rioc->rdmain = NULL; + } + if (rioc->rdmaout) { + qemu_rdma_cleanup(rioc->rdmaout); + g_free(rioc->rdmaout); + rioc->rdmaout = NULL; + } +} + +static void qio_channel_rdma_class_init(ObjectClass *klass, + void *class_data G_GNUC_UNUSED) +{ + QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass); + + ioc_klass->io_writev = qio_channel_rdma_writev; + ioc_klass->io_readv = qio_channel_rdma_readv; + ioc_klass->io_set_blocking = qio_channel_rdma_set_blocking; + ioc_klass->io_close = qio_channel_rdma_close; + ioc_klass->io_create_watch = qio_channel_rdma_create_watch; + ioc_klass->io_set_aio_fd_handler = qio_channel_rdma_set_aio_fd_handler; + ioc_klass->io_shutdown = qio_channel_rdma_shutdown; +} + +static const TypeInfo qio_channel_rdma_info = { + .parent = TYPE_QIO_CHANNEL, + .name = TYPE_QIO_CHANNEL_RDMA, + .instance_size = sizeof(QIOChannelRDMA), + .instance_finalize = qio_channel_rdma_finalize, + .class_init = qio_channel_rdma_class_init, +}; + +static void qio_channel_rdma_register_types(void) +{ + type_register_static(&qio_channel_rdma_info); +} + +type_init(qio_channel_rdma_register_types); + +static QEMUFile *qemu_fopen_rdma(RDMAContext *rdma, const char *mode) +{ + QIOChannelRDMA *rioc; + + if (qemu_file_mode_is_not_valid(mode)) { + return NULL; + } + + rioc = QIO_CHANNEL_RDMA(object_new(TYPE_QIO_CHANNEL_RDMA)); + + if (mode[0] == 'w') { + rioc->file = qemu_fopen_channel_output(QIO_CHANNEL(rioc)); + rioc->rdmaout = rdma; + rioc->rdmain = rdma->return_path; + qemu_file_set_hooks(rioc->file, &rdma_write_hooks); + } else { + rioc->file = qemu_fopen_channel_input(QIO_CHANNEL(rioc)); + rioc->rdmain = rdma; + rioc->rdmaout = rdma->return_path; + qemu_file_set_hooks(rioc->file, &rdma_read_hooks); + } + + return rioc->file; +} + +static void rdma_accept_incoming_migration(void *opaque) +{ + RDMAContext *rdma = opaque; + int ret; + QEMUFile *f; + Error *local_err = NULL; + + trace_qemu_rdma_accept_incoming_migration(); + ret = qemu_rdma_accept(rdma); + + if (ret) { + fprintf(stderr, "RDMA ERROR: Migration initialization failed\n"); + return; + } + + trace_qemu_rdma_accept_incoming_migration_accepted(); + + if (rdma->is_return_path) { + return; + } + + f = qemu_fopen_rdma(rdma, "rb"); + if (f == NULL) { + fprintf(stderr, "RDMA ERROR: could not qemu_fopen_rdma\n"); + qemu_rdma_cleanup(rdma); + return; + } + + rdma->migration_started_on_destination = 1; + migration_fd_process_incoming(f, &local_err); + if (local_err) { + error_reportf_err(local_err, "RDMA ERROR:"); + } +} + +void rdma_start_incoming_migration(const char *host_port, Error **errp) +{ + int ret; + RDMAContext *rdma, *rdma_return_path = NULL; + Error *local_err = NULL; + + trace_rdma_start_incoming_migration(); + + /* Avoid ram_block_discard_disable(), cannot change during migration. */ + if (ram_block_discard_is_required()) { + error_setg(errp, "RDMA: cannot disable RAM discard"); + return; + } + + rdma = qemu_rdma_data_init(host_port, &local_err); + if (rdma == NULL) { + goto err; + } + + ret = qemu_rdma_dest_init(rdma, &local_err); + + if (ret) { + goto err; + } + + trace_rdma_start_incoming_migration_after_dest_init(); + + ret = rdma_listen(rdma->listen_id, 5); + + if (ret) { + ERROR(errp, "listening on socket!"); + goto cleanup_rdma; + } + + trace_rdma_start_incoming_migration_after_rdma_listen(); + + qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration, + NULL, (void *)(intptr_t)rdma); + return; + +cleanup_rdma: + qemu_rdma_cleanup(rdma); +err: + error_propagate(errp, local_err); + if (rdma) { + g_free(rdma->host); + g_free(rdma->host_port); + } + g_free(rdma); + g_free(rdma_return_path); +} + +void rdma_start_outgoing_migration(void *opaque, + const char *host_port, Error **errp) +{ + MigrationState *s = opaque; + RDMAContext *rdma_return_path = NULL; + RDMAContext *rdma; + int ret = 0; + + /* Avoid ram_block_discard_disable(), cannot change during migration. */ + if (ram_block_discard_is_required()) { + error_setg(errp, "RDMA: cannot disable RAM discard"); + return; + } + + rdma = qemu_rdma_data_init(host_port, errp); + if (rdma == NULL) { + goto err; + } + + ret = qemu_rdma_source_init(rdma, + s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL], errp); + + if (ret) { + goto err; + } + + trace_rdma_start_outgoing_migration_after_rdma_source_init(); + ret = qemu_rdma_connect(rdma, errp, false); + + if (ret) { + goto err; + } + + /* RDMA postcopy need a separate queue pair for return path */ + if (migrate_postcopy()) { + rdma_return_path = qemu_rdma_data_init(host_port, errp); + + if (rdma_return_path == NULL) { + goto return_path_err; + } + + ret = qemu_rdma_source_init(rdma_return_path, + s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL], errp); + + if (ret) { + goto return_path_err; + } + + ret = qemu_rdma_connect(rdma_return_path, errp, true); + + if (ret) { + goto return_path_err; + } + + rdma->return_path = rdma_return_path; + rdma_return_path->return_path = rdma; + rdma_return_path->is_return_path = true; + } + + trace_rdma_start_outgoing_migration_after_rdma_connect(); + + s->to_dst_file = qemu_fopen_rdma(rdma, "wb"); + migrate_fd_connect(s, NULL); + return; +return_path_err: + qemu_rdma_cleanup(rdma); +err: + g_free(rdma); + g_free(rdma_return_path); +} diff --git a/migration/rdma.h b/migration/rdma.h new file mode 100644 index 000000000..de2ba09dc --- /dev/null +++ b/migration/rdma.h @@ -0,0 +1,25 @@ +/* + * RDMA protocol and interfaces + * + * Copyright IBM, Corp. 2010-2013 + * Copyright Red Hat, Inc. 2015-2016 + * + * Authors: + * Michael R. Hines <mrhines@us.ibm.com> + * Jiuxing Liu <jl@us.ibm.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or + * later. See the COPYING file in the top-level directory. + * + */ + +#ifndef QEMU_MIGRATION_RDMA_H +#define QEMU_MIGRATION_RDMA_H + +void rdma_start_outgoing_migration(void *opaque, const char *host_port, + Error **errp); + +void rdma_start_incoming_migration(const char *host_port, Error **errp); + +#endif diff --git a/migration/savevm.c b/migration/savevm.c new file mode 100644 index 000000000..d59e976d5 --- /dev/null +++ b/migration/savevm.c @@ -0,0 +1,3303 @@ +/* + * QEMU System Emulator + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2009-2015 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "qemu/osdep.h" +#include "hw/boards.h" +#include "net/net.h" +#include "migration.h" +#include "migration/snapshot.h" +#include "migration/vmstate.h" +#include "migration/misc.h" +#include "migration/register.h" +#include "migration/global_state.h" +#include "ram.h" +#include "qemu-file-channel.h" +#include "qemu-file.h" +#include "savevm.h" +#include "postcopy-ram.h" +#include "qapi/error.h" +#include "qapi/qapi-commands-migration.h" +#include "qapi/qmp/json-writer.h" +#include "qapi/clone-visitor.h" +#include "qapi/qapi-builtin-visit.h" +#include "qapi/qmp/qerror.h" +#include "qemu/error-report.h" +#include "sysemu/cpus.h" +#include "exec/memory.h" +#include "exec/target_page.h" +#include "trace.h" +#include "qemu/iov.h" +#include "qemu/main-loop.h" +#include "block/snapshot.h" +#include "qemu/cutils.h" +#include "io/channel-buffer.h" +#include "io/channel-file.h" +#include "sysemu/replay.h" +#include "sysemu/runstate.h" +#include "sysemu/sysemu.h" +#include "sysemu/xen.h" +#include "migration/colo.h" +#include "qemu/bitmap.h" +#include "net/announce.h" +#include "qemu/yank.h" +#include "yank_functions.h" + +const unsigned int postcopy_ram_discard_version; + +/* Subcommands for QEMU_VM_COMMAND */ +enum qemu_vm_cmd { + MIG_CMD_INVALID = 0, /* Must be 0 */ + MIG_CMD_OPEN_RETURN_PATH, /* Tell the dest to open the Return path */ + MIG_CMD_PING, /* Request a PONG on the RP */ + + MIG_CMD_POSTCOPY_ADVISE, /* Prior to any page transfers, just + warn we might want to do PC */ + MIG_CMD_POSTCOPY_LISTEN, /* Start listening for incoming + pages as it's running. */ + MIG_CMD_POSTCOPY_RUN, /* Start execution */ + + MIG_CMD_POSTCOPY_RAM_DISCARD, /* A list of pages to discard that + were previously sent during + precopy but are dirty. */ + MIG_CMD_PACKAGED, /* Send a wrapped stream within this stream */ + MIG_CMD_ENABLE_COLO, /* Enable COLO */ + MIG_CMD_POSTCOPY_RESUME, /* resume postcopy on dest */ + MIG_CMD_RECV_BITMAP, /* Request for recved bitmap on dst */ + MIG_CMD_MAX +}; + +#define MAX_VM_CMD_PACKAGED_SIZE UINT32_MAX +static struct mig_cmd_args { + ssize_t len; /* -1 = variable */ + const char *name; +} mig_cmd_args[] = { + [MIG_CMD_INVALID] = { .len = -1, .name = "INVALID" }, + [MIG_CMD_OPEN_RETURN_PATH] = { .len = 0, .name = "OPEN_RETURN_PATH" }, + [MIG_CMD_PING] = { .len = sizeof(uint32_t), .name = "PING" }, + [MIG_CMD_POSTCOPY_ADVISE] = { .len = -1, .name = "POSTCOPY_ADVISE" }, + [MIG_CMD_POSTCOPY_LISTEN] = { .len = 0, .name = "POSTCOPY_LISTEN" }, + [MIG_CMD_POSTCOPY_RUN] = { .len = 0, .name = "POSTCOPY_RUN" }, + [MIG_CMD_POSTCOPY_RAM_DISCARD] = { + .len = -1, .name = "POSTCOPY_RAM_DISCARD" }, + [MIG_CMD_POSTCOPY_RESUME] = { .len = 0, .name = "POSTCOPY_RESUME" }, + [MIG_CMD_PACKAGED] = { .len = 4, .name = "PACKAGED" }, + [MIG_CMD_RECV_BITMAP] = { .len = -1, .name = "RECV_BITMAP" }, + [MIG_CMD_MAX] = { .len = -1, .name = "MAX" }, +}; + +/* Note for MIG_CMD_POSTCOPY_ADVISE: + * The format of arguments is depending on postcopy mode: + * - postcopy RAM only + * uint64_t host page size + * uint64_t taget page size + * + * - postcopy RAM and postcopy dirty bitmaps + * format is the same as for postcopy RAM only + * + * - postcopy dirty bitmaps only + * Nothing. Command length field is 0. + * + * Be careful: adding a new postcopy entity with some other parameters should + * not break format self-description ability. Good way is to introduce some + * generic extendable format with an exception for two old entities. + */ + +/***********************************************************/ +/* savevm/loadvm support */ + +static ssize_t block_writev_buffer(void *opaque, struct iovec *iov, int iovcnt, + int64_t pos, Error **errp) +{ + int ret; + QEMUIOVector qiov; + + qemu_iovec_init_external(&qiov, iov, iovcnt); + ret = bdrv_writev_vmstate(opaque, &qiov, pos); + if (ret < 0) { + return ret; + } + + return qiov.size; +} + +static ssize_t block_get_buffer(void *opaque, uint8_t *buf, int64_t pos, + size_t size, Error **errp) +{ + return bdrv_load_vmstate(opaque, buf, pos, size); +} + +static int bdrv_fclose(void *opaque, Error **errp) +{ + return bdrv_flush(opaque); +} + +static const QEMUFileOps bdrv_read_ops = { + .get_buffer = block_get_buffer, + .close = bdrv_fclose +}; + +static const QEMUFileOps bdrv_write_ops = { + .writev_buffer = block_writev_buffer, + .close = bdrv_fclose +}; + +static QEMUFile *qemu_fopen_bdrv(BlockDriverState *bs, int is_writable) +{ + if (is_writable) { + return qemu_fopen_ops(bs, &bdrv_write_ops, false); + } + return qemu_fopen_ops(bs, &bdrv_read_ops, false); +} + + +/* QEMUFile timer support. + * Not in qemu-file.c to not add qemu-timer.c as dependency to qemu-file.c + */ + +void timer_put(QEMUFile *f, QEMUTimer *ts) +{ + uint64_t expire_time; + + expire_time = timer_expire_time_ns(ts); + qemu_put_be64(f, expire_time); +} + +void timer_get(QEMUFile *f, QEMUTimer *ts) +{ + uint64_t expire_time; + + expire_time = qemu_get_be64(f); + if (expire_time != -1) { + timer_mod_ns(ts, expire_time); + } else { + timer_del(ts); + } +} + + +/* VMState timer support. + * Not in vmstate.c to not add qemu-timer.c as dependency to vmstate.c + */ + +static int get_timer(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + QEMUTimer *v = pv; + timer_get(f, v); + return 0; +} + +static int put_timer(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + QEMUTimer *v = pv; + timer_put(f, v); + + return 0; +} + +const VMStateInfo vmstate_info_timer = { + .name = "timer", + .get = get_timer, + .put = put_timer, +}; + + +typedef struct CompatEntry { + char idstr[256]; + int instance_id; +} CompatEntry; + +typedef struct SaveStateEntry { + QTAILQ_ENTRY(SaveStateEntry) entry; + char idstr[256]; + uint32_t instance_id; + int alias_id; + int version_id; + /* version id read from the stream */ + int load_version_id; + int section_id; + /* section id read from the stream */ + int load_section_id; + const SaveVMHandlers *ops; + const VMStateDescription *vmsd; + void *opaque; + CompatEntry *compat; + int is_ram; +} SaveStateEntry; + +typedef struct SaveState { + QTAILQ_HEAD(, SaveStateEntry) handlers; + SaveStateEntry *handler_pri_head[MIG_PRI_MAX + 1]; + int global_section_id; + uint32_t len; + const char *name; + uint32_t target_page_bits; + uint32_t caps_count; + MigrationCapability *capabilities; + QemuUUID uuid; +} SaveState; + +static SaveState savevm_state = { + .handlers = QTAILQ_HEAD_INITIALIZER(savevm_state.handlers), + .handler_pri_head = { [MIG_PRI_DEFAULT ... MIG_PRI_MAX] = NULL }, + .global_section_id = 0, +}; + +static bool should_validate_capability(int capability) +{ + assert(capability >= 0 && capability < MIGRATION_CAPABILITY__MAX); + /* Validate only new capabilities to keep compatibility. */ + switch (capability) { + case MIGRATION_CAPABILITY_X_IGNORE_SHARED: + return true; + default: + return false; + } +} + +static uint32_t get_validatable_capabilities_count(void) +{ + MigrationState *s = migrate_get_current(); + uint32_t result = 0; + int i; + for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { + if (should_validate_capability(i) && s->enabled_capabilities[i]) { + result++; + } + } + return result; +} + +static int configuration_pre_save(void *opaque) +{ + SaveState *state = opaque; + const char *current_name = MACHINE_GET_CLASS(current_machine)->name; + MigrationState *s = migrate_get_current(); + int i, j; + + state->len = strlen(current_name); + state->name = current_name; + state->target_page_bits = qemu_target_page_bits(); + + state->caps_count = get_validatable_capabilities_count(); + state->capabilities = g_renew(MigrationCapability, state->capabilities, + state->caps_count); + for (i = j = 0; i < MIGRATION_CAPABILITY__MAX; i++) { + if (should_validate_capability(i) && s->enabled_capabilities[i]) { + state->capabilities[j++] = i; + } + } + state->uuid = qemu_uuid; + + return 0; +} + +static int configuration_post_save(void *opaque) +{ + SaveState *state = opaque; + + g_free(state->capabilities); + state->capabilities = NULL; + state->caps_count = 0; + return 0; +} + +static int configuration_pre_load(void *opaque) +{ + SaveState *state = opaque; + + /* If there is no target-page-bits subsection it means the source + * predates the variable-target-page-bits support and is using the + * minimum possible value for this CPU. + */ + state->target_page_bits = qemu_target_page_bits_min(); + return 0; +} + +static bool configuration_validate_capabilities(SaveState *state) +{ + bool ret = true; + MigrationState *s = migrate_get_current(); + unsigned long *source_caps_bm; + int i; + + source_caps_bm = bitmap_new(MIGRATION_CAPABILITY__MAX); + for (i = 0; i < state->caps_count; i++) { + MigrationCapability capability = state->capabilities[i]; + set_bit(capability, source_caps_bm); + } + + for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { + bool source_state, target_state; + if (!should_validate_capability(i)) { + continue; + } + source_state = test_bit(i, source_caps_bm); + target_state = s->enabled_capabilities[i]; + if (source_state != target_state) { + error_report("Capability %s is %s, but received capability is %s", + MigrationCapability_str(i), + target_state ? "on" : "off", + source_state ? "on" : "off"); + ret = false; + /* Don't break here to report all failed capabilities */ + } + } + + g_free(source_caps_bm); + return ret; +} + +static int configuration_post_load(void *opaque, int version_id) +{ + SaveState *state = opaque; + const char *current_name = MACHINE_GET_CLASS(current_machine)->name; + int ret = 0; + + if (strncmp(state->name, current_name, state->len) != 0) { + error_report("Machine type received is '%.*s' and local is '%s'", + (int) state->len, state->name, current_name); + ret = -EINVAL; + goto out; + } + + if (state->target_page_bits != qemu_target_page_bits()) { + error_report("Received TARGET_PAGE_BITS is %d but local is %d", + state->target_page_bits, qemu_target_page_bits()); + ret = -EINVAL; + goto out; + } + + if (!configuration_validate_capabilities(state)) { + ret = -EINVAL; + goto out; + } + +out: + g_free((void *)state->name); + state->name = NULL; + state->len = 0; + g_free(state->capabilities); + state->capabilities = NULL; + state->caps_count = 0; + + return ret; +} + +static int get_capability(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + MigrationCapability *capability = pv; + char capability_str[UINT8_MAX + 1]; + uint8_t len; + int i; + + len = qemu_get_byte(f); + qemu_get_buffer(f, (uint8_t *)capability_str, len); + capability_str[len] = '\0'; + for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { + if (!strcmp(MigrationCapability_str(i), capability_str)) { + *capability = i; + return 0; + } + } + error_report("Received unknown capability %s", capability_str); + return -EINVAL; +} + +static int put_capability(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + MigrationCapability *capability = pv; + const char *capability_str = MigrationCapability_str(*capability); + size_t len = strlen(capability_str); + assert(len <= UINT8_MAX); + + qemu_put_byte(f, len); + qemu_put_buffer(f, (uint8_t *)capability_str, len); + return 0; +} + +static const VMStateInfo vmstate_info_capability = { + .name = "capability", + .get = get_capability, + .put = put_capability, +}; + +/* The target-page-bits subsection is present only if the + * target page size is not the same as the default (ie the + * minimum page size for a variable-page-size guest CPU). + * If it is present then it contains the actual target page + * bits for the machine, and migration will fail if the + * two ends don't agree about it. + */ +static bool vmstate_target_page_bits_needed(void *opaque) +{ + return qemu_target_page_bits() + > qemu_target_page_bits_min(); +} + +static const VMStateDescription vmstate_target_page_bits = { + .name = "configuration/target-page-bits", + .version_id = 1, + .minimum_version_id = 1, + .needed = vmstate_target_page_bits_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT32(target_page_bits, SaveState), + VMSTATE_END_OF_LIST() + } +}; + +static bool vmstate_capabilites_needed(void *opaque) +{ + return get_validatable_capabilities_count() > 0; +} + +static const VMStateDescription vmstate_capabilites = { + .name = "configuration/capabilities", + .version_id = 1, + .minimum_version_id = 1, + .needed = vmstate_capabilites_needed, + .fields = (VMStateField[]) { + VMSTATE_UINT32_V(caps_count, SaveState, 1), + VMSTATE_VARRAY_UINT32_ALLOC(capabilities, SaveState, caps_count, 1, + vmstate_info_capability, + MigrationCapability), + VMSTATE_END_OF_LIST() + } +}; + +static bool vmstate_uuid_needed(void *opaque) +{ + return qemu_uuid_set && migrate_validate_uuid(); +} + +static int vmstate_uuid_post_load(void *opaque, int version_id) +{ + SaveState *state = opaque; + char uuid_src[UUID_FMT_LEN + 1]; + char uuid_dst[UUID_FMT_LEN + 1]; + + if (!qemu_uuid_set) { + /* + * It's warning because user might not know UUID in some cases, + * e.g. load an old snapshot + */ + qemu_uuid_unparse(&state->uuid, uuid_src); + warn_report("UUID is received %s, but local uuid isn't set", + uuid_src); + return 0; + } + if (!qemu_uuid_is_equal(&state->uuid, &qemu_uuid)) { + qemu_uuid_unparse(&state->uuid, uuid_src); + qemu_uuid_unparse(&qemu_uuid, uuid_dst); + error_report("UUID received is %s and local is %s", uuid_src, uuid_dst); + return -EINVAL; + } + return 0; +} + +static const VMStateDescription vmstate_uuid = { + .name = "configuration/uuid", + .version_id = 1, + .minimum_version_id = 1, + .needed = vmstate_uuid_needed, + .post_load = vmstate_uuid_post_load, + .fields = (VMStateField[]) { + VMSTATE_UINT8_ARRAY_V(uuid.data, SaveState, sizeof(QemuUUID), 1), + VMSTATE_END_OF_LIST() + } +}; + +static const VMStateDescription vmstate_configuration = { + .name = "configuration", + .version_id = 1, + .pre_load = configuration_pre_load, + .post_load = configuration_post_load, + .pre_save = configuration_pre_save, + .post_save = configuration_post_save, + .fields = (VMStateField[]) { + VMSTATE_UINT32(len, SaveState), + VMSTATE_VBUFFER_ALLOC_UINT32(name, SaveState, 0, NULL, len), + VMSTATE_END_OF_LIST() + }, + .subsections = (const VMStateDescription *[]) { + &vmstate_target_page_bits, + &vmstate_capabilites, + &vmstate_uuid, + NULL + } +}; + +static void dump_vmstate_vmsd(FILE *out_file, + const VMStateDescription *vmsd, int indent, + bool is_subsection); + +static void dump_vmstate_vmsf(FILE *out_file, const VMStateField *field, + int indent) +{ + fprintf(out_file, "%*s{\n", indent, ""); + indent += 2; + fprintf(out_file, "%*s\"field\": \"%s\",\n", indent, "", field->name); + fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "", + field->version_id); + fprintf(out_file, "%*s\"field_exists\": %s,\n", indent, "", + field->field_exists ? "true" : "false"); + fprintf(out_file, "%*s\"size\": %zu", indent, "", field->size); + if (field->vmsd != NULL) { + fprintf(out_file, ",\n"); + dump_vmstate_vmsd(out_file, field->vmsd, indent, false); + } + fprintf(out_file, "\n%*s}", indent - 2, ""); +} + +static void dump_vmstate_vmss(FILE *out_file, + const VMStateDescription **subsection, + int indent) +{ + if (*subsection != NULL) { + dump_vmstate_vmsd(out_file, *subsection, indent, true); + } +} + +static void dump_vmstate_vmsd(FILE *out_file, + const VMStateDescription *vmsd, int indent, + bool is_subsection) +{ + if (is_subsection) { + fprintf(out_file, "%*s{\n", indent, ""); + } else { + fprintf(out_file, "%*s\"%s\": {\n", indent, "", "Description"); + } + indent += 2; + fprintf(out_file, "%*s\"name\": \"%s\",\n", indent, "", vmsd->name); + fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "", + vmsd->version_id); + fprintf(out_file, "%*s\"minimum_version_id\": %d", indent, "", + vmsd->minimum_version_id); + if (vmsd->fields != NULL) { + const VMStateField *field = vmsd->fields; + bool first; + + fprintf(out_file, ",\n%*s\"Fields\": [\n", indent, ""); + first = true; + while (field->name != NULL) { + if (field->flags & VMS_MUST_EXIST) { + /* Ignore VMSTATE_VALIDATE bits; these don't get migrated */ + field++; + continue; + } + if (!first) { + fprintf(out_file, ",\n"); + } + dump_vmstate_vmsf(out_file, field, indent + 2); + field++; + first = false; + } + fprintf(out_file, "\n%*s]", indent, ""); + } + if (vmsd->subsections != NULL) { + const VMStateDescription **subsection = vmsd->subsections; + bool first; + + fprintf(out_file, ",\n%*s\"Subsections\": [\n", indent, ""); + first = true; + while (*subsection != NULL) { + if (!first) { + fprintf(out_file, ",\n"); + } + dump_vmstate_vmss(out_file, subsection, indent + 2); + subsection++; + first = false; + } + fprintf(out_file, "\n%*s]", indent, ""); + } + fprintf(out_file, "\n%*s}", indent - 2, ""); +} + +static void dump_machine_type(FILE *out_file) +{ + MachineClass *mc; + + mc = MACHINE_GET_CLASS(current_machine); + + fprintf(out_file, " \"vmschkmachine\": {\n"); + fprintf(out_file, " \"Name\": \"%s\"\n", mc->name); + fprintf(out_file, " },\n"); +} + +void dump_vmstate_json_to_file(FILE *out_file) +{ + GSList *list, *elt; + bool first; + + fprintf(out_file, "{\n"); + dump_machine_type(out_file); + + first = true; + list = object_class_get_list(TYPE_DEVICE, true); + for (elt = list; elt; elt = elt->next) { + DeviceClass *dc = OBJECT_CLASS_CHECK(DeviceClass, elt->data, + TYPE_DEVICE); + const char *name; + int indent = 2; + + if (!dc->vmsd) { + continue; + } + + if (!first) { + fprintf(out_file, ",\n"); + } + name = object_class_get_name(OBJECT_CLASS(dc)); + fprintf(out_file, "%*s\"%s\": {\n", indent, "", name); + indent += 2; + fprintf(out_file, "%*s\"Name\": \"%s\",\n", indent, "", name); + fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "", + dc->vmsd->version_id); + fprintf(out_file, "%*s\"minimum_version_id\": %d,\n", indent, "", + dc->vmsd->minimum_version_id); + + dump_vmstate_vmsd(out_file, dc->vmsd, indent, false); + + fprintf(out_file, "\n%*s}", indent - 2, ""); + first = false; + } + fprintf(out_file, "\n}\n"); + fclose(out_file); + g_slist_free(list); +} + +static uint32_t calculate_new_instance_id(const char *idstr) +{ + SaveStateEntry *se; + uint32_t instance_id = 0; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (strcmp(idstr, se->idstr) == 0 + && instance_id <= se->instance_id) { + instance_id = se->instance_id + 1; + } + } + /* Make sure we never loop over without being noticed */ + assert(instance_id != VMSTATE_INSTANCE_ID_ANY); + return instance_id; +} + +static int calculate_compat_instance_id(const char *idstr) +{ + SaveStateEntry *se; + int instance_id = 0; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->compat) { + continue; + } + + if (strcmp(idstr, se->compat->idstr) == 0 + && instance_id <= se->compat->instance_id) { + instance_id = se->compat->instance_id + 1; + } + } + return instance_id; +} + +static inline MigrationPriority save_state_priority(SaveStateEntry *se) +{ + if (se->vmsd) { + return se->vmsd->priority; + } + return MIG_PRI_DEFAULT; +} + +static void savevm_state_handler_insert(SaveStateEntry *nse) +{ + MigrationPriority priority = save_state_priority(nse); + SaveStateEntry *se; + int i; + + assert(priority <= MIG_PRI_MAX); + + for (i = priority - 1; i >= 0; i--) { + se = savevm_state.handler_pri_head[i]; + if (se != NULL) { + assert(save_state_priority(se) < priority); + break; + } + } + + if (i >= 0) { + QTAILQ_INSERT_BEFORE(se, nse, entry); + } else { + QTAILQ_INSERT_TAIL(&savevm_state.handlers, nse, entry); + } + + if (savevm_state.handler_pri_head[priority] == NULL) { + savevm_state.handler_pri_head[priority] = nse; + } +} + +static void savevm_state_handler_remove(SaveStateEntry *se) +{ + SaveStateEntry *next; + MigrationPriority priority = save_state_priority(se); + + if (se == savevm_state.handler_pri_head[priority]) { + next = QTAILQ_NEXT(se, entry); + if (next != NULL && save_state_priority(next) == priority) { + savevm_state.handler_pri_head[priority] = next; + } else { + savevm_state.handler_pri_head[priority] = NULL; + } + } + QTAILQ_REMOVE(&savevm_state.handlers, se, entry); +} + +/* TODO: Individual devices generally have very little idea about the rest + of the system, so instance_id should be removed/replaced. + Meanwhile pass -1 as instance_id if you do not already have a clearly + distinguishing id for all instances of your device class. */ +int register_savevm_live(const char *idstr, + uint32_t instance_id, + int version_id, + const SaveVMHandlers *ops, + void *opaque) +{ + SaveStateEntry *se; + + se = g_new0(SaveStateEntry, 1); + se->version_id = version_id; + se->section_id = savevm_state.global_section_id++; + se->ops = ops; + se->opaque = opaque; + se->vmsd = NULL; + /* if this is a live_savem then set is_ram */ + if (ops->save_setup != NULL) { + se->is_ram = 1; + } + + pstrcat(se->idstr, sizeof(se->idstr), idstr); + + if (instance_id == VMSTATE_INSTANCE_ID_ANY) { + se->instance_id = calculate_new_instance_id(se->idstr); + } else { + se->instance_id = instance_id; + } + assert(!se->compat || se->instance_id == 0); + savevm_state_handler_insert(se); + return 0; +} + +void unregister_savevm(VMStateIf *obj, const char *idstr, void *opaque) +{ + SaveStateEntry *se, *new_se; + char id[256] = ""; + + if (obj) { + char *oid = vmstate_if_get_id(obj); + if (oid) { + pstrcpy(id, sizeof(id), oid); + pstrcat(id, sizeof(id), "/"); + g_free(oid); + } + } + pstrcat(id, sizeof(id), idstr); + + QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) { + if (strcmp(se->idstr, id) == 0 && se->opaque == opaque) { + savevm_state_handler_remove(se); + g_free(se->compat); + g_free(se); + } + } +} + +int vmstate_register_with_alias_id(VMStateIf *obj, uint32_t instance_id, + const VMStateDescription *vmsd, + void *opaque, int alias_id, + int required_for_version, + Error **errp) +{ + SaveStateEntry *se; + + /* If this triggers, alias support can be dropped for the vmsd. */ + assert(alias_id == -1 || required_for_version >= vmsd->minimum_version_id); + + se = g_new0(SaveStateEntry, 1); + se->version_id = vmsd->version_id; + se->section_id = savevm_state.global_section_id++; + se->opaque = opaque; + se->vmsd = vmsd; + se->alias_id = alias_id; + + if (obj) { + char *id = vmstate_if_get_id(obj); + if (id) { + if (snprintf(se->idstr, sizeof(se->idstr), "%s/", id) >= + sizeof(se->idstr)) { + error_setg(errp, "Path too long for VMState (%s)", id); + g_free(id); + g_free(se); + + return -1; + } + g_free(id); + + se->compat = g_new0(CompatEntry, 1); + pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), vmsd->name); + se->compat->instance_id = instance_id == VMSTATE_INSTANCE_ID_ANY ? + calculate_compat_instance_id(vmsd->name) : instance_id; + instance_id = VMSTATE_INSTANCE_ID_ANY; + } + } + pstrcat(se->idstr, sizeof(se->idstr), vmsd->name); + + if (instance_id == VMSTATE_INSTANCE_ID_ANY) { + se->instance_id = calculate_new_instance_id(se->idstr); + } else { + se->instance_id = instance_id; + } + assert(!se->compat || se->instance_id == 0); + savevm_state_handler_insert(se); + return 0; +} + +void vmstate_unregister(VMStateIf *obj, const VMStateDescription *vmsd, + void *opaque) +{ + SaveStateEntry *se, *new_se; + + QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) { + if (se->vmsd == vmsd && se->opaque == opaque) { + savevm_state_handler_remove(se); + g_free(se->compat); + g_free(se); + } + } +} + +static int vmstate_load(QEMUFile *f, SaveStateEntry *se) +{ + trace_vmstate_load(se->idstr, se->vmsd ? se->vmsd->name : "(old)"); + if (!se->vmsd) { /* Old style */ + return se->ops->load_state(f, se->opaque, se->load_version_id); + } + return vmstate_load_state(f, se->vmsd, se->opaque, se->load_version_id); +} + +static void vmstate_save_old_style(QEMUFile *f, SaveStateEntry *se, + JSONWriter *vmdesc) +{ + int64_t old_offset, size; + + old_offset = qemu_ftell_fast(f); + se->ops->save_state(f, se->opaque); + size = qemu_ftell_fast(f) - old_offset; + + if (vmdesc) { + json_writer_int64(vmdesc, "size", size); + json_writer_start_array(vmdesc, "fields"); + json_writer_start_object(vmdesc, NULL); + json_writer_str(vmdesc, "name", "data"); + json_writer_int64(vmdesc, "size", size); + json_writer_str(vmdesc, "type", "buffer"); + json_writer_end_object(vmdesc); + json_writer_end_array(vmdesc); + } +} + +static int vmstate_save(QEMUFile *f, SaveStateEntry *se, + JSONWriter *vmdesc) +{ + trace_vmstate_save(se->idstr, se->vmsd ? se->vmsd->name : "(old)"); + if (!se->vmsd) { + vmstate_save_old_style(f, se, vmdesc); + return 0; + } + return vmstate_save_state(f, se->vmsd, se->opaque, vmdesc); +} + +/* + * Write the header for device section (QEMU_VM_SECTION START/END/PART/FULL) + */ +static void save_section_header(QEMUFile *f, SaveStateEntry *se, + uint8_t section_type) +{ + qemu_put_byte(f, section_type); + qemu_put_be32(f, se->section_id); + + if (section_type == QEMU_VM_SECTION_FULL || + section_type == QEMU_VM_SECTION_START) { + /* ID string */ + size_t len = strlen(se->idstr); + qemu_put_byte(f, len); + qemu_put_buffer(f, (uint8_t *)se->idstr, len); + + qemu_put_be32(f, se->instance_id); + qemu_put_be32(f, se->version_id); + } +} + +/* + * Write a footer onto device sections that catches cases misformatted device + * sections. + */ +static void save_section_footer(QEMUFile *f, SaveStateEntry *se) +{ + if (migrate_get_current()->send_section_footer) { + qemu_put_byte(f, QEMU_VM_SECTION_FOOTER); + qemu_put_be32(f, se->section_id); + } +} + +/** + * qemu_savevm_command_send: Send a 'QEMU_VM_COMMAND' type element with the + * command and associated data. + * + * @f: File to send command on + * @command: Command type to send + * @len: Length of associated data + * @data: Data associated with command. + */ +static void qemu_savevm_command_send(QEMUFile *f, + enum qemu_vm_cmd command, + uint16_t len, + uint8_t *data) +{ + trace_savevm_command_send(command, len); + qemu_put_byte(f, QEMU_VM_COMMAND); + qemu_put_be16(f, (uint16_t)command); + qemu_put_be16(f, len); + qemu_put_buffer(f, data, len); + qemu_fflush(f); +} + +void qemu_savevm_send_colo_enable(QEMUFile *f) +{ + trace_savevm_send_colo_enable(); + qemu_savevm_command_send(f, MIG_CMD_ENABLE_COLO, 0, NULL); +} + +void qemu_savevm_send_ping(QEMUFile *f, uint32_t value) +{ + uint32_t buf; + + trace_savevm_send_ping(value); + buf = cpu_to_be32(value); + qemu_savevm_command_send(f, MIG_CMD_PING, sizeof(value), (uint8_t *)&buf); +} + +void qemu_savevm_send_open_return_path(QEMUFile *f) +{ + trace_savevm_send_open_return_path(); + qemu_savevm_command_send(f, MIG_CMD_OPEN_RETURN_PATH, 0, NULL); +} + +/* We have a buffer of data to send; we don't want that all to be loaded + * by the command itself, so the command contains just the length of the + * extra buffer that we then send straight after it. + * TODO: Must be a better way to organise that + * + * Returns: + * 0 on success + * -ve on error + */ +int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len) +{ + uint32_t tmp; + + if (len > MAX_VM_CMD_PACKAGED_SIZE) { + error_report("%s: Unreasonably large packaged state: %zu", + __func__, len); + return -1; + } + + tmp = cpu_to_be32(len); + + trace_qemu_savevm_send_packaged(); + qemu_savevm_command_send(f, MIG_CMD_PACKAGED, 4, (uint8_t *)&tmp); + + qemu_put_buffer(f, buf, len); + + return 0; +} + +/* Send prior to any postcopy transfer */ +void qemu_savevm_send_postcopy_advise(QEMUFile *f) +{ + if (migrate_postcopy_ram()) { + uint64_t tmp[2]; + tmp[0] = cpu_to_be64(ram_pagesize_summary()); + tmp[1] = cpu_to_be64(qemu_target_page_size()); + + trace_qemu_savevm_send_postcopy_advise(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_ADVISE, + 16, (uint8_t *)tmp); + } else { + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_ADVISE, 0, NULL); + } +} + +/* Sent prior to starting the destination running in postcopy, discard pages + * that have already been sent but redirtied on the source. + * CMD_POSTCOPY_RAM_DISCARD consist of: + * byte version (0) + * byte Length of name field (not including 0) + * n x byte RAM block name + * byte 0 terminator (just for safety) + * n x Byte ranges within the named RAMBlock + * be64 Start of the range + * be64 Length + * + * name: RAMBlock name that these entries are part of + * len: Number of page entries + * start_list: 'len' addresses + * length_list: 'len' addresses + * + */ +void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name, + uint16_t len, + uint64_t *start_list, + uint64_t *length_list) +{ + uint8_t *buf; + uint16_t tmplen; + uint16_t t; + size_t name_len = strlen(name); + + trace_qemu_savevm_send_postcopy_ram_discard(name, len); + assert(name_len < 256); + buf = g_malloc0(1 + 1 + name_len + 1 + (8 + 8) * len); + buf[0] = postcopy_ram_discard_version; + buf[1] = name_len; + memcpy(buf + 2, name, name_len); + tmplen = 2 + name_len; + buf[tmplen++] = '\0'; + + for (t = 0; t < len; t++) { + stq_be_p(buf + tmplen, start_list[t]); + tmplen += 8; + stq_be_p(buf + tmplen, length_list[t]); + tmplen += 8; + } + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RAM_DISCARD, tmplen, buf); + g_free(buf); +} + +/* Get the destination into a state where it can receive postcopy data. */ +void qemu_savevm_send_postcopy_listen(QEMUFile *f) +{ + trace_savevm_send_postcopy_listen(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_LISTEN, 0, NULL); +} + +/* Kick the destination into running */ +void qemu_savevm_send_postcopy_run(QEMUFile *f) +{ + trace_savevm_send_postcopy_run(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RUN, 0, NULL); +} + +void qemu_savevm_send_postcopy_resume(QEMUFile *f) +{ + trace_savevm_send_postcopy_resume(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RESUME, 0, NULL); +} + +void qemu_savevm_send_recv_bitmap(QEMUFile *f, char *block_name) +{ + size_t len; + char buf[256]; + + trace_savevm_send_recv_bitmap(block_name); + + buf[0] = len = strlen(block_name); + memcpy(buf + 1, block_name, len); + + qemu_savevm_command_send(f, MIG_CMD_RECV_BITMAP, len + 1, (uint8_t *)buf); +} + +bool qemu_savevm_state_blocked(Error **errp) +{ + SaveStateEntry *se; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (se->vmsd && se->vmsd->unmigratable) { + error_setg(errp, "State blocked by non-migratable device '%s'", + se->idstr); + return true; + } + } + return false; +} + +void qemu_savevm_non_migratable_list(strList **reasons) +{ + SaveStateEntry *se; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (se->vmsd && se->vmsd->unmigratable) { + QAPI_LIST_PREPEND(*reasons, + g_strdup_printf("non-migratable device: %s", + se->idstr)); + } + } +} + +void qemu_savevm_state_header(QEMUFile *f) +{ + trace_savevm_state_header(); + qemu_put_be32(f, QEMU_VM_FILE_MAGIC); + qemu_put_be32(f, QEMU_VM_FILE_VERSION); + + if (migrate_get_current()->send_configuration) { + qemu_put_byte(f, QEMU_VM_CONFIGURATION); + vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0); + } +} + +bool qemu_savevm_state_guest_unplug_pending(void) +{ + SaveStateEntry *se; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (se->vmsd && se->vmsd->dev_unplug_pending && + se->vmsd->dev_unplug_pending(se->opaque)) { + return true; + } + } + + return false; +} + +void qemu_savevm_state_setup(QEMUFile *f) +{ + SaveStateEntry *se; + Error *local_err = NULL; + int ret; + + trace_savevm_state_setup(); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->save_setup) { + continue; + } + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + save_section_header(f, se, QEMU_VM_SECTION_START); + + ret = se->ops->save_setup(f, se->opaque); + save_section_footer(f, se); + if (ret < 0) { + qemu_file_set_error(f, ret); + break; + } + } + + if (precopy_notify(PRECOPY_NOTIFY_SETUP, &local_err)) { + error_report_err(local_err); + } +} + +int qemu_savevm_state_resume_prepare(MigrationState *s) +{ + SaveStateEntry *se; + int ret; + + trace_savevm_state_resume_prepare(); + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->resume_prepare) { + continue; + } + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + ret = se->ops->resume_prepare(s, se->opaque); + if (ret < 0) { + return ret; + } + } + + return 0; +} + +/* + * this function has three return values: + * negative: there was one error, and we have -errno. + * 0 : We haven't finished, caller have to go again + * 1 : We have finished, we can go to complete phase + */ +int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy) +{ + SaveStateEntry *se; + int ret = 1; + + trace_savevm_state_iterate(); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->save_live_iterate) { + continue; + } + if (se->ops->is_active && + !se->ops->is_active(se->opaque)) { + continue; + } + if (se->ops->is_active_iterate && + !se->ops->is_active_iterate(se->opaque)) { + continue; + } + /* + * In the postcopy phase, any device that doesn't know how to + * do postcopy should have saved it's state in the _complete + * call that's already run, it might get confused if we call + * iterate afterwards. + */ + if (postcopy && + !(se->ops->has_postcopy && se->ops->has_postcopy(se->opaque))) { + continue; + } + if (qemu_file_rate_limit(f)) { + return 0; + } + trace_savevm_section_start(se->idstr, se->section_id); + + save_section_header(f, se, QEMU_VM_SECTION_PART); + + ret = se->ops->save_live_iterate(f, se->opaque); + trace_savevm_section_end(se->idstr, se->section_id, ret); + save_section_footer(f, se); + + if (ret < 0) { + error_report("failed to save SaveStateEntry with id(name): %d(%s)", + se->section_id, se->idstr); + qemu_file_set_error(f, ret); + } + if (ret <= 0) { + /* Do not proceed to the next vmstate before this one reported + completion of the current stage. This serializes the migration + and reduces the probability that a faster changing state is + synchronized over and over again. */ + break; + } + } + return ret; +} + +static bool should_send_vmdesc(void) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + bool in_postcopy = migration_in_postcopy(); + return !machine->suppress_vmdesc && !in_postcopy; +} + +/* + * Calls the save_live_complete_postcopy methods + * causing the last few pages to be sent immediately and doing any associated + * cleanup. + * Note postcopy also calls qemu_savevm_state_complete_precopy to complete + * all the other devices, but that happens at the point we switch to postcopy. + */ +void qemu_savevm_state_complete_postcopy(QEMUFile *f) +{ + SaveStateEntry *se; + int ret; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->save_live_complete_postcopy) { + continue; + } + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + trace_savevm_section_start(se->idstr, se->section_id); + /* Section type */ + qemu_put_byte(f, QEMU_VM_SECTION_END); + qemu_put_be32(f, se->section_id); + + ret = se->ops->save_live_complete_postcopy(f, se->opaque); + trace_savevm_section_end(se->idstr, se->section_id, ret); + save_section_footer(f, se); + if (ret < 0) { + qemu_file_set_error(f, ret); + return; + } + } + + qemu_put_byte(f, QEMU_VM_EOF); + qemu_fflush(f); +} + +static +int qemu_savevm_state_complete_precopy_iterable(QEMUFile *f, bool in_postcopy) +{ + SaveStateEntry *se; + int ret; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || + (in_postcopy && se->ops->has_postcopy && + se->ops->has_postcopy(se->opaque)) || + !se->ops->save_live_complete_precopy) { + continue; + } + + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + trace_savevm_section_start(se->idstr, se->section_id); + + save_section_header(f, se, QEMU_VM_SECTION_END); + + ret = se->ops->save_live_complete_precopy(f, se->opaque); + trace_savevm_section_end(se->idstr, se->section_id, ret); + save_section_footer(f, se); + if (ret < 0) { + qemu_file_set_error(f, ret); + return -1; + } + } + + return 0; +} + +int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, + bool in_postcopy, + bool inactivate_disks) +{ + g_autoptr(JSONWriter) vmdesc = NULL; + int vmdesc_len; + SaveStateEntry *se; + int ret; + + vmdesc = json_writer_new(false); + json_writer_start_object(vmdesc, NULL); + json_writer_int64(vmdesc, "page_size", qemu_target_page_size()); + json_writer_start_array(vmdesc, "devices"); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + + if ((!se->ops || !se->ops->save_state) && !se->vmsd) { + continue; + } + if (se->vmsd && !vmstate_save_needed(se->vmsd, se->opaque)) { + trace_savevm_section_skip(se->idstr, se->section_id); + continue; + } + + trace_savevm_section_start(se->idstr, se->section_id); + + json_writer_start_object(vmdesc, NULL); + json_writer_str(vmdesc, "name", se->idstr); + json_writer_int64(vmdesc, "instance_id", se->instance_id); + + save_section_header(f, se, QEMU_VM_SECTION_FULL); + ret = vmstate_save(f, se, vmdesc); + if (ret) { + qemu_file_set_error(f, ret); + return ret; + } + trace_savevm_section_end(se->idstr, se->section_id, 0); + save_section_footer(f, se); + + json_writer_end_object(vmdesc); + } + + if (inactivate_disks) { + /* Inactivate before sending QEMU_VM_EOF so that the + * bdrv_invalidate_cache_all() on the other end won't fail. */ + ret = bdrv_inactivate_all(); + if (ret) { + error_report("%s: bdrv_inactivate_all() failed (%d)", + __func__, ret); + qemu_file_set_error(f, ret); + return ret; + } + } + if (!in_postcopy) { + /* Postcopy stream will still be going */ + qemu_put_byte(f, QEMU_VM_EOF); + } + + json_writer_end_array(vmdesc); + json_writer_end_object(vmdesc); + vmdesc_len = strlen(json_writer_get(vmdesc)); + + if (should_send_vmdesc()) { + qemu_put_byte(f, QEMU_VM_VMDESCRIPTION); + qemu_put_be32(f, vmdesc_len); + qemu_put_buffer(f, (uint8_t *)json_writer_get(vmdesc), vmdesc_len); + } + + return 0; +} + +int qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only, + bool inactivate_disks) +{ + int ret; + Error *local_err = NULL; + bool in_postcopy = migration_in_postcopy(); + + if (precopy_notify(PRECOPY_NOTIFY_COMPLETE, &local_err)) { + error_report_err(local_err); + } + + trace_savevm_state_complete_precopy(); + + cpu_synchronize_all_states(); + + if (!in_postcopy || iterable_only) { + ret = qemu_savevm_state_complete_precopy_iterable(f, in_postcopy); + if (ret) { + return ret; + } + } + + if (iterable_only) { + goto flush; + } + + ret = qemu_savevm_state_complete_precopy_non_iterable(f, in_postcopy, + inactivate_disks); + if (ret) { + return ret; + } + +flush: + qemu_fflush(f); + return 0; +} + +/* Give an estimate of the amount left to be transferred, + * the result is split into the amount for units that can and + * for units that can't do postcopy. + */ +void qemu_savevm_state_pending(QEMUFile *f, uint64_t threshold_size, + uint64_t *res_precopy_only, + uint64_t *res_compatible, + uint64_t *res_postcopy_only) +{ + SaveStateEntry *se; + + *res_precopy_only = 0; + *res_compatible = 0; + *res_postcopy_only = 0; + + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->save_live_pending) { + continue; + } + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + se->ops->save_live_pending(f, se->opaque, threshold_size, + res_precopy_only, res_compatible, + res_postcopy_only); + } +} + +void qemu_savevm_state_cleanup(void) +{ + SaveStateEntry *se; + Error *local_err = NULL; + + if (precopy_notify(PRECOPY_NOTIFY_CLEANUP, &local_err)) { + error_report_err(local_err); + } + + trace_savevm_state_cleanup(); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (se->ops && se->ops->save_cleanup) { + se->ops->save_cleanup(se->opaque); + } + } +} + +static int qemu_savevm_state(QEMUFile *f, Error **errp) +{ + int ret; + MigrationState *ms = migrate_get_current(); + MigrationStatus status; + + if (migration_is_running(ms->state)) { + error_setg(errp, QERR_MIGRATION_ACTIVE); + return -EINVAL; + } + + if (migrate_use_block()) { + error_setg(errp, "Block migration and snapshots are incompatible"); + return -EINVAL; + } + + migrate_init(ms); + memset(&ram_counters, 0, sizeof(ram_counters)); + memset(&compression_counters, 0, sizeof(compression_counters)); + ms->to_dst_file = f; + + qemu_mutex_unlock_iothread(); + qemu_savevm_state_header(f); + qemu_savevm_state_setup(f); + qemu_mutex_lock_iothread(); + + while (qemu_file_get_error(f) == 0) { + if (qemu_savevm_state_iterate(f, false) > 0) { + break; + } + } + + ret = qemu_file_get_error(f); + if (ret == 0) { + qemu_savevm_state_complete_precopy(f, false, false); + ret = qemu_file_get_error(f); + } + qemu_savevm_state_cleanup(); + if (ret != 0) { + error_setg_errno(errp, -ret, "Error while writing VM state"); + } + + if (ret != 0) { + status = MIGRATION_STATUS_FAILED; + } else { + status = MIGRATION_STATUS_COMPLETED; + } + migrate_set_state(&ms->state, MIGRATION_STATUS_SETUP, status); + + /* f is outer parameter, it should not stay in global migration state after + * this function finished */ + ms->to_dst_file = NULL; + + return ret; +} + +void qemu_savevm_live_state(QEMUFile *f) +{ + /* save QEMU_VM_SECTION_END section */ + qemu_savevm_state_complete_precopy(f, true, false); + qemu_put_byte(f, QEMU_VM_EOF); +} + +int qemu_save_device_state(QEMUFile *f) +{ + SaveStateEntry *se; + + if (!migration_in_colo_state()) { + qemu_put_be32(f, QEMU_VM_FILE_MAGIC); + qemu_put_be32(f, QEMU_VM_FILE_VERSION); + } + cpu_synchronize_all_states(); + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + int ret; + + if (se->is_ram) { + continue; + } + if ((!se->ops || !se->ops->save_state) && !se->vmsd) { + continue; + } + if (se->vmsd && !vmstate_save_needed(se->vmsd, se->opaque)) { + continue; + } + + save_section_header(f, se, QEMU_VM_SECTION_FULL); + + ret = vmstate_save(f, se, NULL); + if (ret) { + return ret; + } + + save_section_footer(f, se); + } + + qemu_put_byte(f, QEMU_VM_EOF); + + return qemu_file_get_error(f); +} + +static SaveStateEntry *find_se(const char *idstr, uint32_t instance_id) +{ + SaveStateEntry *se; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!strcmp(se->idstr, idstr) && + (instance_id == se->instance_id || + instance_id == se->alias_id)) + return se; + /* Migrating from an older version? */ + if (strstr(se->idstr, idstr) && se->compat) { + if (!strcmp(se->compat->idstr, idstr) && + (instance_id == se->compat->instance_id || + instance_id == se->alias_id)) + return se; + } + } + return NULL; +} + +enum LoadVMExitCodes { + /* Allow a command to quit all layers of nested loadvm loops */ + LOADVM_QUIT = 1, +}; + +/* ------ incoming postcopy messages ------ */ +/* 'advise' arrives before any transfers just to tell us that a postcopy + * *might* happen - it might be skipped if precopy transferred everything + * quickly. + */ +static int loadvm_postcopy_handle_advise(MigrationIncomingState *mis, + uint16_t len) +{ + PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_ADVISE); + uint64_t remote_pagesize_summary, local_pagesize_summary, remote_tps; + Error *local_err = NULL; + + trace_loadvm_postcopy_handle_advise(); + if (ps != POSTCOPY_INCOMING_NONE) { + error_report("CMD_POSTCOPY_ADVISE in wrong postcopy state (%d)", ps); + return -1; + } + + switch (len) { + case 0: + if (migrate_postcopy_ram()) { + error_report("RAM postcopy is enabled but have 0 byte advise"); + return -EINVAL; + } + return 0; + case 8 + 8: + if (!migrate_postcopy_ram()) { + error_report("RAM postcopy is disabled but have 16 byte advise"); + return -EINVAL; + } + break; + default: + error_report("CMD_POSTCOPY_ADVISE invalid length (%d)", len); + return -EINVAL; + } + + if (!postcopy_ram_supported_by_host(mis)) { + postcopy_state_set(POSTCOPY_INCOMING_NONE); + return -1; + } + + remote_pagesize_summary = qemu_get_be64(mis->from_src_file); + local_pagesize_summary = ram_pagesize_summary(); + + if (remote_pagesize_summary != local_pagesize_summary) { + /* + * This detects two potential causes of mismatch: + * a) A mismatch in host page sizes + * Some combinations of mismatch are probably possible but it gets + * a bit more complicated. In particular we need to place whole + * host pages on the dest at once, and we need to ensure that we + * handle dirtying to make sure we never end up sending part of + * a hostpage on it's own. + * b) The use of different huge page sizes on source/destination + * a more fine grain test is performed during RAM block migration + * but this test here causes a nice early clear failure, and + * also fails when passed to an older qemu that doesn't + * do huge pages. + */ + error_report("Postcopy needs matching RAM page sizes (s=%" PRIx64 + " d=%" PRIx64 ")", + remote_pagesize_summary, local_pagesize_summary); + return -1; + } + + remote_tps = qemu_get_be64(mis->from_src_file); + if (remote_tps != qemu_target_page_size()) { + /* + * Again, some differences could be dealt with, but for now keep it + * simple. + */ + error_report("Postcopy needs matching target page sizes (s=%d d=%zd)", + (int)remote_tps, qemu_target_page_size()); + return -1; + } + + if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_ADVISE, &local_err)) { + error_report_err(local_err); + return -1; + } + + if (ram_postcopy_incoming_init(mis)) { + return -1; + } + + return 0; +} + +/* After postcopy we will be told to throw some pages away since they're + * dirty and will have to be demand fetched. Must happen before CPU is + * started. + * There can be 0..many of these messages, each encoding multiple pages. + */ +static int loadvm_postcopy_ram_handle_discard(MigrationIncomingState *mis, + uint16_t len) +{ + int tmp; + char ramid[256]; + PostcopyState ps = postcopy_state_get(); + + trace_loadvm_postcopy_ram_handle_discard(); + + switch (ps) { + case POSTCOPY_INCOMING_ADVISE: + /* 1st discard */ + tmp = postcopy_ram_prepare_discard(mis); + if (tmp) { + return tmp; + } + break; + + case POSTCOPY_INCOMING_DISCARD: + /* Expected state */ + break; + + default: + error_report("CMD_POSTCOPY_RAM_DISCARD in wrong postcopy state (%d)", + ps); + return -1; + } + /* We're expecting a + * Version (0) + * a RAM ID string (length byte, name, 0 term) + * then at least 1 16 byte chunk + */ + if (len < (1 + 1 + 1 + 1 + 2 * 8)) { + error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len); + return -1; + } + + tmp = qemu_get_byte(mis->from_src_file); + if (tmp != postcopy_ram_discard_version) { + error_report("CMD_POSTCOPY_RAM_DISCARD invalid version (%d)", tmp); + return -1; + } + + if (!qemu_get_counted_string(mis->from_src_file, ramid)) { + error_report("CMD_POSTCOPY_RAM_DISCARD Failed to read RAMBlock ID"); + return -1; + } + tmp = qemu_get_byte(mis->from_src_file); + if (tmp != 0) { + error_report("CMD_POSTCOPY_RAM_DISCARD missing nil (%d)", tmp); + return -1; + } + + len -= 3 + strlen(ramid); + if (len % 16) { + error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len); + return -1; + } + trace_loadvm_postcopy_ram_handle_discard_header(ramid, len); + while (len) { + uint64_t start_addr, block_length; + start_addr = qemu_get_be64(mis->from_src_file); + block_length = qemu_get_be64(mis->from_src_file); + + len -= 16; + int ret = ram_discard_range(ramid, start_addr, block_length); + if (ret) { + return ret; + } + } + trace_loadvm_postcopy_ram_handle_discard_end(); + + return 0; +} + +/* + * Triggered by a postcopy_listen command; this thread takes over reading + * the input stream, leaving the main thread free to carry on loading the rest + * of the device state (from RAM). + * (TODO:This could do with being in a postcopy file - but there again it's + * just another input loop, not that postcopy specific) + */ +static void *postcopy_ram_listen_thread(void *opaque) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + QEMUFile *f = mis->from_src_file; + int load_res; + MigrationState *migr = migrate_get_current(); + + object_ref(OBJECT(migr)); + + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + qemu_sem_post(&mis->listen_thread_sem); + trace_postcopy_ram_listen_thread_start(); + + rcu_register_thread(); + /* + * Because we're a thread and not a coroutine we can't yield + * in qemu_file, and thus we must be blocking now. + */ + qemu_file_set_blocking(f, true); + load_res = qemu_loadvm_state_main(f, mis); + + /* + * This is tricky, but, mis->from_src_file can change after it + * returns, when postcopy recovery happened. In the future, we may + * want a wrapper for the QEMUFile handle. + */ + f = mis->from_src_file; + + /* And non-blocking again so we don't block in any cleanup */ + qemu_file_set_blocking(f, false); + + trace_postcopy_ram_listen_thread_exit(); + if (load_res < 0) { + qemu_file_set_error(f, load_res); + dirty_bitmap_mig_cancel_incoming(); + if (postcopy_state_get() == POSTCOPY_INCOMING_RUNNING && + !migrate_postcopy_ram() && migrate_dirty_bitmaps()) + { + error_report("%s: loadvm failed during postcopy: %d. All states " + "are migrated except dirty bitmaps. Some dirty " + "bitmaps may be lost, and present migrated dirty " + "bitmaps are correctly migrated and valid.", + __func__, load_res); + load_res = 0; /* prevent further exit() */ + } else { + error_report("%s: loadvm failed: %d", __func__, load_res); + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_FAILED); + } + } + if (load_res >= 0) { + /* + * This looks good, but it's possible that the device loading in the + * main thread hasn't finished yet, and so we might not be in 'RUN' + * state yet; wait for the end of the main thread. + */ + qemu_event_wait(&mis->main_thread_load_event); + } + postcopy_ram_incoming_cleanup(mis); + + if (load_res < 0) { + /* + * If something went wrong then we have a bad state so exit; + * depending how far we got it might be possible at this point + * to leave the guest running and fire MCEs for pages that never + * arrived as a desperate recovery step. + */ + rcu_unregister_thread(); + exit(EXIT_FAILURE); + } + + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_COMPLETED); + /* + * If everything has worked fine, then the main thread has waited + * for us to start, and we're the last use of the mis. + * (If something broke then qemu will have to exit anyway since it's + * got a bad migration state). + */ + migration_incoming_state_destroy(); + qemu_loadvm_state_cleanup(); + + rcu_unregister_thread(); + mis->have_listen_thread = false; + postcopy_state_set(POSTCOPY_INCOMING_END); + + object_unref(OBJECT(migr)); + + return NULL; +} + +/* After this message we must be able to immediately receive postcopy data */ +static int loadvm_postcopy_handle_listen(MigrationIncomingState *mis) +{ + PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_LISTENING); + trace_loadvm_postcopy_handle_listen(); + Error *local_err = NULL; + + if (ps != POSTCOPY_INCOMING_ADVISE && ps != POSTCOPY_INCOMING_DISCARD) { + error_report("CMD_POSTCOPY_LISTEN in wrong postcopy state (%d)", ps); + return -1; + } + if (ps == POSTCOPY_INCOMING_ADVISE) { + /* + * A rare case, we entered listen without having to do any discards, + * so do the setup that's normally done at the time of the 1st discard. + */ + if (migrate_postcopy_ram()) { + postcopy_ram_prepare_discard(mis); + } + } + + /* + * Sensitise RAM - can now generate requests for blocks that don't exist + * However, at this point the CPU shouldn't be running, and the IO + * shouldn't be doing anything yet so don't actually expect requests + */ + if (migrate_postcopy_ram()) { + if (postcopy_ram_incoming_setup(mis)) { + postcopy_ram_incoming_cleanup(mis); + return -1; + } + } + + if (postcopy_notify(POSTCOPY_NOTIFY_INBOUND_LISTEN, &local_err)) { + error_report_err(local_err); + return -1; + } + + mis->have_listen_thread = true; + /* Start up the listening thread and wait for it to signal ready */ + qemu_sem_init(&mis->listen_thread_sem, 0); + qemu_thread_create(&mis->listen_thread, "postcopy/listen", + postcopy_ram_listen_thread, NULL, + QEMU_THREAD_DETACHED); + qemu_sem_wait(&mis->listen_thread_sem); + qemu_sem_destroy(&mis->listen_thread_sem); + + return 0; +} + +static void loadvm_postcopy_handle_run_bh(void *opaque) +{ + Error *local_err = NULL; + MigrationIncomingState *mis = opaque; + + /* TODO we should move all of this lot into postcopy_ram.c or a shared code + * in migration.c + */ + cpu_synchronize_all_post_init(); + + qemu_announce_self(&mis->announce_timer, migrate_announce_params()); + + /* Make sure all file formats flush their mutable metadata. + * If we get an error here, just don't restart the VM yet. */ + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + local_err = NULL; + autostart = false; + } + + trace_loadvm_postcopy_handle_run_cpu_sync(); + + trace_loadvm_postcopy_handle_run_vmstart(); + + dirty_bitmap_mig_before_vm_start(); + + if (autostart) { + /* Hold onto your hats, starting the CPU */ + vm_start(); + } else { + /* leave it paused and let management decide when to start the CPU */ + runstate_set(RUN_STATE_PAUSED); + } + + qemu_bh_delete(mis->bh); +} + +/* After all discards we can start running and asking for pages */ +static int loadvm_postcopy_handle_run(MigrationIncomingState *mis) +{ + PostcopyState ps = postcopy_state_get(); + + trace_loadvm_postcopy_handle_run(); + if (ps != POSTCOPY_INCOMING_LISTENING) { + error_report("CMD_POSTCOPY_RUN in wrong postcopy state (%d)", ps); + return -1; + } + + postcopy_state_set(POSTCOPY_INCOMING_RUNNING); + mis->bh = qemu_bh_new(loadvm_postcopy_handle_run_bh, mis); + qemu_bh_schedule(mis->bh); + + /* We need to finish reading the stream from the package + * and also stop reading anything more from the stream that loaded the + * package (since it's now being read by the listener thread). + * LOADVM_QUIT will quit all the layers of nested loadvm loops. + */ + return LOADVM_QUIT; +} + +/* We must be with page_request_mutex held */ +static gboolean postcopy_sync_page_req(gpointer key, gpointer value, + gpointer data) +{ + MigrationIncomingState *mis = data; + void *host_addr = (void *) key; + ram_addr_t rb_offset; + RAMBlock *rb; + int ret; + + rb = qemu_ram_block_from_host(host_addr, true, &rb_offset); + if (!rb) { + /* + * This should _never_ happen. However be nice for a migrating VM to + * not crash/assert. Post an error (note: intended to not use *_once + * because we do want to see all the illegal addresses; and this can + * never be triggered by the guest so we're safe) and move on next. + */ + error_report("%s: illegal host addr %p", __func__, host_addr); + /* Try the next entry */ + return FALSE; + } + + ret = migrate_send_rp_message_req_pages(mis, rb, rb_offset); + if (ret) { + /* Please refer to above comment. */ + error_report("%s: send rp message failed for addr %p", + __func__, host_addr); + return FALSE; + } + + trace_postcopy_page_req_sync(host_addr); + + return FALSE; +} + +static void migrate_send_rp_req_pages_pending(MigrationIncomingState *mis) +{ + WITH_QEMU_LOCK_GUARD(&mis->page_request_mutex) { + g_tree_foreach(mis->page_requested, postcopy_sync_page_req, mis); + } +} + +static int loadvm_postcopy_handle_resume(MigrationIncomingState *mis) +{ + if (mis->state != MIGRATION_STATUS_POSTCOPY_RECOVER) { + error_report("%s: illegal resume received", __func__); + /* Don't fail the load, only for this. */ + return 0; + } + + /* + * Reset the last_rb before we resend any page req to source again, since + * the source should have it reset already. + */ + mis->last_rb = NULL; + + /* + * This means source VM is ready to resume the postcopy migration. + */ + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_RECOVER, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + + trace_loadvm_postcopy_handle_resume(); + + /* Tell source that "we are ready" */ + migrate_send_rp_resume_ack(mis, MIGRATION_RESUME_ACK_VALUE); + + /* + * After a postcopy recovery, the source should have lost the postcopy + * queue, or potentially the requested pages could have been lost during + * the network down phase. Let's re-sync with the source VM by re-sending + * all the pending pages that we eagerly need, so these threads won't get + * blocked too long due to the recovery. + * + * Without this procedure, the faulted destination VM threads (waiting for + * page requests right before the postcopy is interrupted) can keep hanging + * until the pages are sent by the source during the background copying of + * pages, or another thread faulted on the same address accidentally. + */ + migrate_send_rp_req_pages_pending(mis); + + /* + * It's time to switch state and release the fault thread to continue + * service page faults. Note that this should be explicitly after the + * above call to migrate_send_rp_req_pages_pending(). In short: + * migrate_send_rp_message_req_pages() is not thread safe, yet. + */ + qemu_sem_post(&mis->postcopy_pause_sem_fault); + + return 0; +} + +/** + * Immediately following this command is a blob of data containing an embedded + * chunk of migration stream; read it and load it. + * + * @mis: Incoming state + * @length: Length of packaged data to read + * + * Returns: Negative values on error + * + */ +static int loadvm_handle_cmd_packaged(MigrationIncomingState *mis) +{ + int ret; + size_t length; + QIOChannelBuffer *bioc; + + length = qemu_get_be32(mis->from_src_file); + trace_loadvm_handle_cmd_packaged(length); + + if (length > MAX_VM_CMD_PACKAGED_SIZE) { + error_report("Unreasonably large packaged state: %zu", length); + return -1; + } + + bioc = qio_channel_buffer_new(length); + qio_channel_set_name(QIO_CHANNEL(bioc), "migration-loadvm-buffer"); + ret = qemu_get_buffer(mis->from_src_file, + bioc->data, + length); + if (ret != length) { + object_unref(OBJECT(bioc)); + error_report("CMD_PACKAGED: Buffer receive fail ret=%d length=%zu", + ret, length); + return (ret < 0) ? ret : -EAGAIN; + } + bioc->usage += length; + trace_loadvm_handle_cmd_packaged_received(ret); + + QEMUFile *packf = qemu_fopen_channel_input(QIO_CHANNEL(bioc)); + + ret = qemu_loadvm_state_main(packf, mis); + trace_loadvm_handle_cmd_packaged_main(ret); + qemu_fclose(packf); + object_unref(OBJECT(bioc)); + + return ret; +} + +/* + * Handle request that source requests for recved_bitmap on + * destination. Payload format: + * + * len (1 byte) + ramblock_name (<255 bytes) + */ +static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis, + uint16_t len) +{ + QEMUFile *file = mis->from_src_file; + RAMBlock *rb; + char block_name[256]; + size_t cnt; + + cnt = qemu_get_counted_string(file, block_name); + if (!cnt) { + error_report("%s: failed to read block name", __func__); + return -EINVAL; + } + + /* Validate before using the data */ + if (qemu_file_get_error(file)) { + return qemu_file_get_error(file); + } + + if (len != cnt + 1) { + error_report("%s: invalid payload length (%d)", __func__, len); + return -EINVAL; + } + + rb = qemu_ram_block_by_name(block_name); + if (!rb) { + error_report("%s: block '%s' not found", __func__, block_name); + return -EINVAL; + } + + migrate_send_rp_recv_bitmap(mis, block_name); + + trace_loadvm_handle_recv_bitmap(block_name); + + return 0; +} + +static int loadvm_process_enable_colo(MigrationIncomingState *mis) +{ + int ret = migration_incoming_enable_colo(); + + if (!ret) { + ret = colo_init_ram_cache(); + if (ret) { + migration_incoming_disable_colo(); + } + } + return ret; +} + +/* + * Process an incoming 'QEMU_VM_COMMAND' + * 0 just a normal return + * LOADVM_QUIT All good, but exit the loop + * <0 Error + */ +static int loadvm_process_command(QEMUFile *f) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + uint16_t cmd; + uint16_t len; + uint32_t tmp32; + + cmd = qemu_get_be16(f); + len = qemu_get_be16(f); + + /* Check validity before continue processing of cmds */ + if (qemu_file_get_error(f)) { + return qemu_file_get_error(f); + } + + trace_loadvm_process_command(cmd, len); + if (cmd >= MIG_CMD_MAX || cmd == MIG_CMD_INVALID) { + error_report("MIG_CMD 0x%x unknown (len 0x%x)", cmd, len); + return -EINVAL; + } + + if (mig_cmd_args[cmd].len != -1 && mig_cmd_args[cmd].len != len) { + error_report("%s received with bad length - expecting %zu, got %d", + mig_cmd_args[cmd].name, + (size_t)mig_cmd_args[cmd].len, len); + return -ERANGE; + } + + switch (cmd) { + case MIG_CMD_OPEN_RETURN_PATH: + if (mis->to_src_file) { + error_report("CMD_OPEN_RETURN_PATH called when RP already open"); + /* Not really a problem, so don't give up */ + return 0; + } + mis->to_src_file = qemu_file_get_return_path(f); + if (!mis->to_src_file) { + error_report("CMD_OPEN_RETURN_PATH failed"); + return -1; + } + break; + + case MIG_CMD_PING: + tmp32 = qemu_get_be32(f); + trace_loadvm_process_command_ping(tmp32); + if (!mis->to_src_file) { + error_report("CMD_PING (0x%x) received with no return path", + tmp32); + return -1; + } + migrate_send_rp_pong(mis, tmp32); + break; + + case MIG_CMD_PACKAGED: + return loadvm_handle_cmd_packaged(mis); + + case MIG_CMD_POSTCOPY_ADVISE: + return loadvm_postcopy_handle_advise(mis, len); + + case MIG_CMD_POSTCOPY_LISTEN: + return loadvm_postcopy_handle_listen(mis); + + case MIG_CMD_POSTCOPY_RUN: + return loadvm_postcopy_handle_run(mis); + + case MIG_CMD_POSTCOPY_RAM_DISCARD: + return loadvm_postcopy_ram_handle_discard(mis, len); + + case MIG_CMD_POSTCOPY_RESUME: + return loadvm_postcopy_handle_resume(mis); + + case MIG_CMD_RECV_BITMAP: + return loadvm_handle_recv_bitmap(mis, len); + + case MIG_CMD_ENABLE_COLO: + return loadvm_process_enable_colo(mis); + } + + return 0; +} + +/* + * Read a footer off the wire and check that it matches the expected section + * + * Returns: true if the footer was good + * false if there is a problem (and calls error_report to say why) + */ +static bool check_section_footer(QEMUFile *f, SaveStateEntry *se) +{ + int ret; + uint8_t read_mark; + uint32_t read_section_id; + + if (!migrate_get_current()->send_section_footer) { + /* No footer to check */ + return true; + } + + read_mark = qemu_get_byte(f); + + ret = qemu_file_get_error(f); + if (ret) { + error_report("%s: Read section footer failed: %d", + __func__, ret); + return false; + } + + if (read_mark != QEMU_VM_SECTION_FOOTER) { + error_report("Missing section footer for %s", se->idstr); + return false; + } + + read_section_id = qemu_get_be32(f); + if (read_section_id != se->load_section_id) { + error_report("Mismatched section id in footer for %s -" + " read 0x%x expected 0x%x", + se->idstr, read_section_id, se->load_section_id); + return false; + } + + /* All good */ + return true; +} + +static int +qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis) +{ + uint32_t instance_id, version_id, section_id; + SaveStateEntry *se; + char idstr[256]; + int ret; + + /* Read section start */ + section_id = qemu_get_be32(f); + if (!qemu_get_counted_string(f, idstr)) { + error_report("Unable to read ID string for section %u", + section_id); + return -EINVAL; + } + instance_id = qemu_get_be32(f); + version_id = qemu_get_be32(f); + + ret = qemu_file_get_error(f); + if (ret) { + error_report("%s: Failed to read instance/version ID: %d", + __func__, ret); + return ret; + } + + trace_qemu_loadvm_state_section_startfull(section_id, idstr, + instance_id, version_id); + /* Find savevm section */ + se = find_se(idstr, instance_id); + if (se == NULL) { + error_report("Unknown savevm section or instance '%s' %"PRIu32". " + "Make sure that your current VM setup matches your " + "saved VM setup, including any hotplugged devices", + idstr, instance_id); + return -EINVAL; + } + + /* Validate version */ + if (version_id > se->version_id) { + error_report("savevm: unsupported version %d for '%s' v%d", + version_id, idstr, se->version_id); + return -EINVAL; + } + se->load_version_id = version_id; + se->load_section_id = section_id; + + /* Validate if it is a device's state */ + if (xen_enabled() && se->is_ram) { + error_report("loadvm: %s RAM loading not allowed on Xen", idstr); + return -EINVAL; + } + + ret = vmstate_load(f, se); + if (ret < 0) { + error_report("error while loading state for instance 0x%"PRIx32" of" + " device '%s'", instance_id, idstr); + return ret; + } + if (!check_section_footer(f, se)) { + return -EINVAL; + } + + return 0; +} + +static int +qemu_loadvm_section_part_end(QEMUFile *f, MigrationIncomingState *mis) +{ + uint32_t section_id; + SaveStateEntry *se; + int ret; + + section_id = qemu_get_be32(f); + + ret = qemu_file_get_error(f); + if (ret) { + error_report("%s: Failed to read section ID: %d", + __func__, ret); + return ret; + } + + trace_qemu_loadvm_state_section_partend(section_id); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (se->load_section_id == section_id) { + break; + } + } + if (se == NULL) { + error_report("Unknown savevm section %d", section_id); + return -EINVAL; + } + + ret = vmstate_load(f, se); + if (ret < 0) { + error_report("error while loading state section id %d(%s)", + section_id, se->idstr); + return ret; + } + if (!check_section_footer(f, se)) { + return -EINVAL; + } + + return 0; +} + +static int qemu_loadvm_state_header(QEMUFile *f) +{ + unsigned int v; + int ret; + + v = qemu_get_be32(f); + if (v != QEMU_VM_FILE_MAGIC) { + error_report("Not a migration stream"); + return -EINVAL; + } + + v = qemu_get_be32(f); + if (v == QEMU_VM_FILE_VERSION_COMPAT) { + error_report("SaveVM v2 format is obsolete and don't work anymore"); + return -ENOTSUP; + } + if (v != QEMU_VM_FILE_VERSION) { + error_report("Unsupported migration stream version"); + return -ENOTSUP; + } + + if (migrate_get_current()->send_configuration) { + if (qemu_get_byte(f) != QEMU_VM_CONFIGURATION) { + error_report("Configuration section missing"); + qemu_loadvm_state_cleanup(); + return -EINVAL; + } + ret = vmstate_load_state(f, &vmstate_configuration, &savevm_state, 0); + + if (ret) { + qemu_loadvm_state_cleanup(); + return ret; + } + } + return 0; +} + +static int qemu_loadvm_state_setup(QEMUFile *f) +{ + SaveStateEntry *se; + int ret; + + trace_loadvm_state_setup(); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->load_setup) { + continue; + } + if (se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + + ret = se->ops->load_setup(f, se->opaque); + if (ret < 0) { + qemu_file_set_error(f, ret); + error_report("Load state of device %s failed", se->idstr); + return ret; + } + } + return 0; +} + +void qemu_loadvm_state_cleanup(void) +{ + SaveStateEntry *se; + + trace_loadvm_state_cleanup(); + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (se->ops && se->ops->load_cleanup) { + se->ops->load_cleanup(se->opaque); + } + } +} + +/* Return true if we should continue the migration, or false. */ +static bool postcopy_pause_incoming(MigrationIncomingState *mis) +{ + trace_postcopy_pause_incoming(); + + assert(migrate_postcopy_ram()); + + /* Clear the triggered bit to allow one recovery */ + mis->postcopy_recover_triggered = false; + + /* + * Unregister yank with either from/to src would work, since ioc behind it + * is the same + */ + migration_ioc_unregister_yank_from_file(mis->from_src_file); + + assert(mis->from_src_file); + qemu_file_shutdown(mis->from_src_file); + qemu_fclose(mis->from_src_file); + mis->from_src_file = NULL; + + assert(mis->to_src_file); + qemu_file_shutdown(mis->to_src_file); + qemu_mutex_lock(&mis->rp_mutex); + qemu_fclose(mis->to_src_file); + mis->to_src_file = NULL; + qemu_mutex_unlock(&mis->rp_mutex); + + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_POSTCOPY_PAUSED); + + /* Notify the fault thread for the invalidated file handle */ + postcopy_fault_thread_notify(mis); + + error_report("Detected IO failure for postcopy. " + "Migration paused."); + + while (mis->state == MIGRATION_STATUS_POSTCOPY_PAUSED) { + qemu_sem_wait(&mis->postcopy_pause_sem_dst); + } + + trace_postcopy_pause_incoming_continued(); + + return true; +} + +int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis) +{ + uint8_t section_type; + int ret = 0; + +retry: + while (true) { + section_type = qemu_get_byte(f); + + if (qemu_file_get_error(f)) { + ret = qemu_file_get_error(f); + break; + } + + trace_qemu_loadvm_state_section(section_type); + switch (section_type) { + case QEMU_VM_SECTION_START: + case QEMU_VM_SECTION_FULL: + ret = qemu_loadvm_section_start_full(f, mis); + if (ret < 0) { + goto out; + } + break; + case QEMU_VM_SECTION_PART: + case QEMU_VM_SECTION_END: + ret = qemu_loadvm_section_part_end(f, mis); + if (ret < 0) { + goto out; + } + break; + case QEMU_VM_COMMAND: + ret = loadvm_process_command(f); + trace_qemu_loadvm_state_section_command(ret); + if ((ret < 0) || (ret == LOADVM_QUIT)) { + goto out; + } + break; + case QEMU_VM_EOF: + /* This is the end of migration */ + goto out; + default: + error_report("Unknown savevm section type %d", section_type); + ret = -EINVAL; + goto out; + } + } + +out: + if (ret < 0) { + qemu_file_set_error(f, ret); + + /* Cancel bitmaps incoming regardless of recovery */ + dirty_bitmap_mig_cancel_incoming(); + + /* + * If we are during an active postcopy, then we pause instead + * of bail out to at least keep the VM's dirty data. Note + * that POSTCOPY_INCOMING_LISTENING stage is still not enough, + * during which we're still receiving device states and we + * still haven't yet started the VM on destination. + * + * Only RAM postcopy supports recovery. Still, if RAM postcopy is + * enabled, canceled bitmaps postcopy will not affect RAM postcopy + * recovering. + */ + if (postcopy_state_get() == POSTCOPY_INCOMING_RUNNING && + migrate_postcopy_ram() && postcopy_pause_incoming(mis)) { + /* Reset f to point to the newly created channel */ + f = mis->from_src_file; + goto retry; + } + } + return ret; +} + +int qemu_loadvm_state(QEMUFile *f) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + Error *local_err = NULL; + int ret; + + if (qemu_savevm_state_blocked(&local_err)) { + error_report_err(local_err); + return -EINVAL; + } + + ret = qemu_loadvm_state_header(f); + if (ret) { + return ret; + } + + if (qemu_loadvm_state_setup(f) != 0) { + return -EINVAL; + } + + cpu_synchronize_all_pre_loadvm(); + + ret = qemu_loadvm_state_main(f, mis); + qemu_event_set(&mis->main_thread_load_event); + + trace_qemu_loadvm_state_post_main(ret); + + if (mis->have_listen_thread) { + /* Listen thread still going, can't clean up yet */ + return ret; + } + + if (ret == 0) { + ret = qemu_file_get_error(f); + } + + /* + * Try to read in the VMDESC section as well, so that dumping tools that + * intercept our migration stream have the chance to see it. + */ + + /* We've got to be careful; if we don't read the data and just shut the fd + * then the sender can error if we close while it's still sending. + * We also mustn't read data that isn't there; some transports (RDMA) + * will stall waiting for that data when the source has already closed. + */ + if (ret == 0 && should_send_vmdesc()) { + uint8_t *buf; + uint32_t size; + uint8_t section_type = qemu_get_byte(f); + + if (section_type != QEMU_VM_VMDESCRIPTION) { + error_report("Expected vmdescription section, but got %d", + section_type); + /* + * It doesn't seem worth failing at this point since + * we apparently have an otherwise valid VM state + */ + } else { + buf = g_malloc(0x1000); + size = qemu_get_be32(f); + + while (size > 0) { + uint32_t read_chunk = MIN(size, 0x1000); + qemu_get_buffer(f, buf, read_chunk); + size -= read_chunk; + } + g_free(buf); + } + } + + qemu_loadvm_state_cleanup(); + cpu_synchronize_all_post_init(); + + return ret; +} + +int qemu_load_device_state(QEMUFile *f) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + int ret; + + /* Load QEMU_VM_SECTION_FULL section */ + ret = qemu_loadvm_state_main(f, mis); + if (ret < 0) { + error_report("Failed to load device state: %d", ret); + return ret; + } + + cpu_synchronize_all_post_init(); + return 0; +} + +bool save_snapshot(const char *name, bool overwrite, const char *vmstate, + bool has_devices, strList *devices, Error **errp) +{ + BlockDriverState *bs; + QEMUSnapshotInfo sn1, *sn = &sn1; + int ret = -1, ret2; + QEMUFile *f; + int saved_vm_running; + uint64_t vm_state_size; + g_autoptr(GDateTime) now = g_date_time_new_now_local(); + AioContext *aio_context; + + if (migration_is_blocked(errp)) { + return false; + } + + if (!replay_can_snapshot()) { + error_setg(errp, "Record/replay does not allow making snapshot " + "right now. Try once more later."); + return false; + } + + if (!bdrv_all_can_snapshot(has_devices, devices, errp)) { + return false; + } + + /* Delete old snapshots of the same name */ + if (name) { + if (overwrite) { + if (bdrv_all_delete_snapshot(name, has_devices, + devices, errp) < 0) { + return false; + } + } else { + ret2 = bdrv_all_has_snapshot(name, has_devices, devices, errp); + if (ret2 < 0) { + return false; + } + if (ret2 == 1) { + error_setg(errp, + "Snapshot '%s' already exists in one or more devices", + name); + return false; + } + } + } + + bs = bdrv_all_find_vmstate_bs(vmstate, has_devices, devices, errp); + if (bs == NULL) { + return false; + } + aio_context = bdrv_get_aio_context(bs); + + saved_vm_running = runstate_is_running(); + + ret = global_state_store(); + if (ret) { + error_setg(errp, "Error saving global state"); + return false; + } + vm_stop(RUN_STATE_SAVE_VM); + + bdrv_drain_all_begin(); + + aio_context_acquire(aio_context); + + memset(sn, 0, sizeof(*sn)); + + /* fill auxiliary fields */ + sn->date_sec = g_date_time_to_unix(now); + sn->date_nsec = g_date_time_get_microsecond(now) * 1000; + sn->vm_clock_nsec = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + if (replay_mode != REPLAY_MODE_NONE) { + sn->icount = replay_get_current_icount(); + } else { + sn->icount = -1ULL; + } + + if (name) { + pstrcpy(sn->name, sizeof(sn->name), name); + } else { + g_autofree char *autoname = g_date_time_format(now, "vm-%Y%m%d%H%M%S"); + pstrcpy(sn->name, sizeof(sn->name), autoname); + } + + /* save the VM state */ + f = qemu_fopen_bdrv(bs, 1); + if (!f) { + error_setg(errp, "Could not open VM state file"); + goto the_end; + } + ret = qemu_savevm_state(f, errp); + vm_state_size = qemu_ftell(f); + ret2 = qemu_fclose(f); + if (ret < 0) { + goto the_end; + } + if (ret2 < 0) { + ret = ret2; + goto the_end; + } + + /* The bdrv_all_create_snapshot() call that follows acquires the AioContext + * for itself. BDRV_POLL_WHILE() does not support nested locking because + * it only releases the lock once. Therefore synchronous I/O will deadlock + * unless we release the AioContext before bdrv_all_create_snapshot(). + */ + aio_context_release(aio_context); + aio_context = NULL; + + ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, + has_devices, devices, errp); + if (ret < 0) { + bdrv_all_delete_snapshot(sn->name, has_devices, devices, NULL); + goto the_end; + } + + ret = 0; + + the_end: + if (aio_context) { + aio_context_release(aio_context); + } + + bdrv_drain_all_end(); + + if (saved_vm_running) { + vm_start(); + } + return ret == 0; +} + +void qmp_xen_save_devices_state(const char *filename, bool has_live, bool live, + Error **errp) +{ + QEMUFile *f; + QIOChannelFile *ioc; + int saved_vm_running; + int ret; + + if (!has_live) { + /* live default to true so old version of Xen tool stack can have a + * successful live migration */ + live = true; + } + + saved_vm_running = runstate_is_running(); + vm_stop(RUN_STATE_SAVE_VM); + global_state_store_running(); + + ioc = qio_channel_file_new_path(filename, O_WRONLY | O_CREAT | O_TRUNC, + 0660, errp); + if (!ioc) { + goto the_end; + } + qio_channel_set_name(QIO_CHANNEL(ioc), "migration-xen-save-state"); + f = qemu_fopen_channel_output(QIO_CHANNEL(ioc)); + object_unref(OBJECT(ioc)); + ret = qemu_save_device_state(f); + if (ret < 0 || qemu_fclose(f) < 0) { + error_setg(errp, QERR_IO_ERROR); + } else { + /* libxl calls the QMP command "stop" before calling + * "xen-save-devices-state" and in case of migration failure, libxl + * would call "cont". + * So call bdrv_inactivate_all (release locks) here to let the other + * side of the migration take control of the images. + */ + if (live && !saved_vm_running) { + ret = bdrv_inactivate_all(); + if (ret) { + error_setg(errp, "%s: bdrv_inactivate_all() failed (%d)", + __func__, ret); + } + } + } + + the_end: + if (saved_vm_running) { + vm_start(); + } +} + +void qmp_xen_load_devices_state(const char *filename, Error **errp) +{ + QEMUFile *f; + QIOChannelFile *ioc; + int ret; + + /* Guest must be paused before loading the device state; the RAM state + * will already have been loaded by xc + */ + if (runstate_is_running()) { + error_setg(errp, "Cannot update device state while vm is running"); + return; + } + vm_stop(RUN_STATE_RESTORE_VM); + + ioc = qio_channel_file_new_path(filename, O_RDONLY | O_BINARY, 0, errp); + if (!ioc) { + return; + } + qio_channel_set_name(QIO_CHANNEL(ioc), "migration-xen-load-state"); + f = qemu_fopen_channel_input(QIO_CHANNEL(ioc)); + object_unref(OBJECT(ioc)); + + ret = qemu_loadvm_state(f); + qemu_fclose(f); + if (ret < 0) { + error_setg(errp, QERR_IO_ERROR); + } + migration_incoming_state_destroy(); +} + +bool load_snapshot(const char *name, const char *vmstate, + bool has_devices, strList *devices, Error **errp) +{ + BlockDriverState *bs_vm_state; + QEMUSnapshotInfo sn; + QEMUFile *f; + int ret; + AioContext *aio_context; + MigrationIncomingState *mis = migration_incoming_get_current(); + + if (!bdrv_all_can_snapshot(has_devices, devices, errp)) { + return false; + } + ret = bdrv_all_has_snapshot(name, has_devices, devices, errp); + if (ret < 0) { + return false; + } + if (ret == 0) { + error_setg(errp, "Snapshot '%s' does not exist in one or more devices", + name); + return false; + } + + bs_vm_state = bdrv_all_find_vmstate_bs(vmstate, has_devices, devices, errp); + if (!bs_vm_state) { + return false; + } + aio_context = bdrv_get_aio_context(bs_vm_state); + + /* Don't even try to load empty VM states */ + aio_context_acquire(aio_context); + ret = bdrv_snapshot_find(bs_vm_state, &sn, name); + aio_context_release(aio_context); + if (ret < 0) { + return false; + } else if (sn.vm_state_size == 0) { + error_setg(errp, "This is a disk-only snapshot. Revert to it " + " offline using qemu-img"); + return false; + } + + /* + * Flush the record/replay queue. Now the VM state is going + * to change. Therefore we don't need to preserve its consistency + */ + replay_flush_events(); + + /* Flush all IO requests so they don't interfere with the new state. */ + bdrv_drain_all_begin(); + + ret = bdrv_all_goto_snapshot(name, has_devices, devices, errp); + if (ret < 0) { + goto err_drain; + } + + /* restore the VM state */ + f = qemu_fopen_bdrv(bs_vm_state, 0); + if (!f) { + error_setg(errp, "Could not open VM state file"); + goto err_drain; + } + + qemu_system_reset(SHUTDOWN_CAUSE_NONE); + mis->from_src_file = f; + + if (!yank_register_instance(MIGRATION_YANK_INSTANCE, errp)) { + ret = -EINVAL; + goto err_drain; + } + aio_context_acquire(aio_context); + ret = qemu_loadvm_state(f); + migration_incoming_state_destroy(); + aio_context_release(aio_context); + + bdrv_drain_all_end(); + + if (ret < 0) { + error_setg(errp, "Error %d while loading VM state", ret); + return false; + } + + return true; + +err_drain: + bdrv_drain_all_end(); + return false; +} + +bool delete_snapshot(const char *name, bool has_devices, + strList *devices, Error **errp) +{ + if (!bdrv_all_can_snapshot(has_devices, devices, errp)) { + return false; + } + + if (bdrv_all_delete_snapshot(name, has_devices, devices, errp) < 0) { + return false; + } + + return true; +} + +void vmstate_register_ram(MemoryRegion *mr, DeviceState *dev) +{ + qemu_ram_set_idstr(mr->ram_block, + memory_region_name(mr), dev); + qemu_ram_set_migratable(mr->ram_block); +} + +void vmstate_unregister_ram(MemoryRegion *mr, DeviceState *dev) +{ + qemu_ram_unset_idstr(mr->ram_block); + qemu_ram_unset_migratable(mr->ram_block); +} + +void vmstate_register_ram_global(MemoryRegion *mr) +{ + vmstate_register_ram(mr, NULL); +} + +bool vmstate_check_only_migratable(const VMStateDescription *vmsd) +{ + /* check needed if --only-migratable is specified */ + if (!only_migratable) { + return true; + } + + return !(vmsd && vmsd->unmigratable); +} + +typedef struct SnapshotJob { + Job common; + char *tag; + char *vmstate; + strList *devices; + Coroutine *co; + Error **errp; + bool ret; +} SnapshotJob; + +static void qmp_snapshot_job_free(SnapshotJob *s) +{ + g_free(s->tag); + g_free(s->vmstate); + qapi_free_strList(s->devices); +} + + +static void snapshot_load_job_bh(void *opaque) +{ + Job *job = opaque; + SnapshotJob *s = container_of(job, SnapshotJob, common); + int orig_vm_running; + + job_progress_set_remaining(&s->common, 1); + + orig_vm_running = runstate_is_running(); + vm_stop(RUN_STATE_RESTORE_VM); + + s->ret = load_snapshot(s->tag, s->vmstate, true, s->devices, s->errp); + if (s->ret && orig_vm_running) { + vm_start(); + } + + job_progress_update(&s->common, 1); + + qmp_snapshot_job_free(s); + aio_co_wake(s->co); +} + +static void snapshot_save_job_bh(void *opaque) +{ + Job *job = opaque; + SnapshotJob *s = container_of(job, SnapshotJob, common); + + job_progress_set_remaining(&s->common, 1); + s->ret = save_snapshot(s->tag, false, s->vmstate, + true, s->devices, s->errp); + job_progress_update(&s->common, 1); + + qmp_snapshot_job_free(s); + aio_co_wake(s->co); +} + +static void snapshot_delete_job_bh(void *opaque) +{ + Job *job = opaque; + SnapshotJob *s = container_of(job, SnapshotJob, common); + + job_progress_set_remaining(&s->common, 1); + s->ret = delete_snapshot(s->tag, true, s->devices, s->errp); + job_progress_update(&s->common, 1); + + qmp_snapshot_job_free(s); + aio_co_wake(s->co); +} + +static int coroutine_fn snapshot_save_job_run(Job *job, Error **errp) +{ + SnapshotJob *s = container_of(job, SnapshotJob, common); + s->errp = errp; + s->co = qemu_coroutine_self(); + aio_bh_schedule_oneshot(qemu_get_aio_context(), + snapshot_save_job_bh, job); + qemu_coroutine_yield(); + return s->ret ? 0 : -1; +} + +static int coroutine_fn snapshot_load_job_run(Job *job, Error **errp) +{ + SnapshotJob *s = container_of(job, SnapshotJob, common); + s->errp = errp; + s->co = qemu_coroutine_self(); + aio_bh_schedule_oneshot(qemu_get_aio_context(), + snapshot_load_job_bh, job); + qemu_coroutine_yield(); + return s->ret ? 0 : -1; +} + +static int coroutine_fn snapshot_delete_job_run(Job *job, Error **errp) +{ + SnapshotJob *s = container_of(job, SnapshotJob, common); + s->errp = errp; + s->co = qemu_coroutine_self(); + aio_bh_schedule_oneshot(qemu_get_aio_context(), + snapshot_delete_job_bh, job); + qemu_coroutine_yield(); + return s->ret ? 0 : -1; +} + + +static const JobDriver snapshot_load_job_driver = { + .instance_size = sizeof(SnapshotJob), + .job_type = JOB_TYPE_SNAPSHOT_LOAD, + .run = snapshot_load_job_run, +}; + +static const JobDriver snapshot_save_job_driver = { + .instance_size = sizeof(SnapshotJob), + .job_type = JOB_TYPE_SNAPSHOT_SAVE, + .run = snapshot_save_job_run, +}; + +static const JobDriver snapshot_delete_job_driver = { + .instance_size = sizeof(SnapshotJob), + .job_type = JOB_TYPE_SNAPSHOT_DELETE, + .run = snapshot_delete_job_run, +}; + + +void qmp_snapshot_save(const char *job_id, + const char *tag, + const char *vmstate, + strList *devices, + Error **errp) +{ + SnapshotJob *s; + + s = job_create(job_id, &snapshot_save_job_driver, NULL, + qemu_get_aio_context(), JOB_MANUAL_DISMISS, + NULL, NULL, errp); + if (!s) { + return; + } + + s->tag = g_strdup(tag); + s->vmstate = g_strdup(vmstate); + s->devices = QAPI_CLONE(strList, devices); + + job_start(&s->common); +} + +void qmp_snapshot_load(const char *job_id, + const char *tag, + const char *vmstate, + strList *devices, + Error **errp) +{ + SnapshotJob *s; + + s = job_create(job_id, &snapshot_load_job_driver, NULL, + qemu_get_aio_context(), JOB_MANUAL_DISMISS, + NULL, NULL, errp); + if (!s) { + return; + } + + s->tag = g_strdup(tag); + s->vmstate = g_strdup(vmstate); + s->devices = QAPI_CLONE(strList, devices); + + job_start(&s->common); +} + +void qmp_snapshot_delete(const char *job_id, + const char *tag, + strList *devices, + Error **errp) +{ + SnapshotJob *s; + + s = job_create(job_id, &snapshot_delete_job_driver, NULL, + qemu_get_aio_context(), JOB_MANUAL_DISMISS, + NULL, NULL, errp); + if (!s) { + return; + } + + s->tag = g_strdup(tag); + s->devices = QAPI_CLONE(strList, devices); + + job_start(&s->common); +} diff --git a/migration/savevm.h b/migration/savevm.h new file mode 100644 index 000000000..6461342cb --- /dev/null +++ b/migration/savevm.h @@ -0,0 +1,71 @@ +/* + * QEMU save vm functions + * + * Copyright (c) 2003-2008 Fabrice Bellard + * Copyright (c) 2009-2017 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef MIGRATION_SAVEVM_H +#define MIGRATION_SAVEVM_H + +#define QEMU_VM_FILE_MAGIC 0x5145564d +#define QEMU_VM_FILE_VERSION_COMPAT 0x00000002 +#define QEMU_VM_FILE_VERSION 0x00000003 + +#define QEMU_VM_EOF 0x00 +#define QEMU_VM_SECTION_START 0x01 +#define QEMU_VM_SECTION_PART 0x02 +#define QEMU_VM_SECTION_END 0x03 +#define QEMU_VM_SECTION_FULL 0x04 +#define QEMU_VM_SUBSECTION 0x05 +#define QEMU_VM_VMDESCRIPTION 0x06 +#define QEMU_VM_CONFIGURATION 0x07 +#define QEMU_VM_COMMAND 0x08 +#define QEMU_VM_SECTION_FOOTER 0x7e + +bool qemu_savevm_state_blocked(Error **errp); +void qemu_savevm_non_migratable_list(strList **reasons); +void qemu_savevm_state_setup(QEMUFile *f); +bool qemu_savevm_state_guest_unplug_pending(void); +int qemu_savevm_state_resume_prepare(MigrationState *s); +void qemu_savevm_state_header(QEMUFile *f); +int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy); +void qemu_savevm_state_cleanup(void); +void qemu_savevm_state_complete_postcopy(QEMUFile *f); +int qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only, + bool inactivate_disks); +void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size, + uint64_t *res_precopy_only, + uint64_t *res_compatible, + uint64_t *res_postcopy_only); +void qemu_savevm_send_ping(QEMUFile *f, uint32_t value); +void qemu_savevm_send_open_return_path(QEMUFile *f); +int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len); +void qemu_savevm_send_postcopy_advise(QEMUFile *f); +void qemu_savevm_send_postcopy_listen(QEMUFile *f); +void qemu_savevm_send_postcopy_run(QEMUFile *f); +void qemu_savevm_send_postcopy_resume(QEMUFile *f); +void qemu_savevm_send_recv_bitmap(QEMUFile *f, char *block_name); + +void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name, + uint16_t len, + uint64_t *start_list, + uint64_t *length_list); +void qemu_savevm_send_colo_enable(QEMUFile *f); +void qemu_savevm_live_state(QEMUFile *f); +int qemu_save_device_state(QEMUFile *f); + +int qemu_loadvm_state(QEMUFile *f); +void qemu_loadvm_state_cleanup(void); +int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis); +int qemu_load_device_state(QEMUFile *f); +int qemu_savevm_state_complete_precopy_non_iterable(QEMUFile *f, + bool in_postcopy, bool inactivate_disks); + +#endif diff --git a/migration/socket.c b/migration/socket.c new file mode 100644 index 000000000..05705a32d --- /dev/null +++ b/migration/socket.c @@ -0,0 +1,196 @@ +/* + * QEMU live migration via socket + * + * Copyright Red Hat, Inc. 2009-2016 + * + * Authors: + * Chris Lalancette <clalance@redhat.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "qemu/cutils.h" + +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "channel.h" +#include "socket.h" +#include "migration.h" +#include "qemu-file.h" +#include "io/channel-socket.h" +#include "io/net-listener.h" +#include "trace.h" + + +struct SocketOutgoingArgs { + SocketAddress *saddr; +} outgoing_args; + +void socket_send_channel_create(QIOTaskFunc f, void *data) +{ + QIOChannelSocket *sioc = qio_channel_socket_new(); + qio_channel_socket_connect_async(sioc, outgoing_args.saddr, + f, data, NULL, NULL); +} + +int socket_send_channel_destroy(QIOChannel *send) +{ + /* Remove channel */ + object_unref(OBJECT(send)); + if (outgoing_args.saddr) { + qapi_free_SocketAddress(outgoing_args.saddr); + outgoing_args.saddr = NULL; + } + return 0; +} + +struct SocketConnectData { + MigrationState *s; + char *hostname; +}; + +static void socket_connect_data_free(void *opaque) +{ + struct SocketConnectData *data = opaque; + if (!data) { + return; + } + g_free(data->hostname); + g_free(data); +} + +static void socket_outgoing_migration(QIOTask *task, + gpointer opaque) +{ + struct SocketConnectData *data = opaque; + QIOChannel *sioc = QIO_CHANNEL(qio_task_get_source(task)); + Error *err = NULL; + + if (qio_task_propagate_error(task, &err)) { + trace_migration_socket_outgoing_error(error_get_pretty(err)); + } else { + trace_migration_socket_outgoing_connected(data->hostname); + } + migration_channel_connect(data->s, sioc, data->hostname, err); + object_unref(OBJECT(sioc)); +} + +static void +socket_start_outgoing_migration_internal(MigrationState *s, + SocketAddress *saddr, + Error **errp) +{ + QIOChannelSocket *sioc = qio_channel_socket_new(); + struct SocketConnectData *data = g_new0(struct SocketConnectData, 1); + + data->s = s; + + /* in case previous migration leaked it */ + qapi_free_SocketAddress(outgoing_args.saddr); + outgoing_args.saddr = saddr; + + if (saddr->type == SOCKET_ADDRESS_TYPE_INET) { + data->hostname = g_strdup(saddr->u.inet.host); + } + + qio_channel_set_name(QIO_CHANNEL(sioc), "migration-socket-outgoing"); + qio_channel_socket_connect_async(sioc, + saddr, + socket_outgoing_migration, + data, + socket_connect_data_free, + NULL); +} + +void socket_start_outgoing_migration(MigrationState *s, + const char *str, + Error **errp) +{ + Error *err = NULL; + SocketAddress *saddr = socket_parse(str, &err); + if (!err) { + socket_start_outgoing_migration_internal(s, saddr, &err); + } + error_propagate(errp, err); +} + +static void socket_accept_incoming_migration(QIONetListener *listener, + QIOChannelSocket *cioc, + gpointer opaque) +{ + trace_migration_socket_incoming_accepted(); + + if (migration_has_all_channels()) { + error_report("%s: Extra incoming migration connection; ignoring", + __func__); + return; + } + + qio_channel_set_name(QIO_CHANNEL(cioc), "migration-socket-incoming"); + migration_channel_process_incoming(QIO_CHANNEL(cioc)); +} + +static void +socket_incoming_migration_end(void *opaque) +{ + QIONetListener *listener = opaque; + + qio_net_listener_disconnect(listener); + object_unref(OBJECT(listener)); +} + +static void +socket_start_incoming_migration_internal(SocketAddress *saddr, + Error **errp) +{ + QIONetListener *listener = qio_net_listener_new(); + MigrationIncomingState *mis = migration_incoming_get_current(); + size_t i; + int num = 1; + + qio_net_listener_set_name(listener, "migration-socket-listener"); + + if (migrate_use_multifd()) { + num = migrate_multifd_channels(); + } + + if (qio_net_listener_open_sync(listener, saddr, num, errp) < 0) { + object_unref(OBJECT(listener)); + return; + } + + mis->transport_data = listener; + mis->transport_cleanup = socket_incoming_migration_end; + + qio_net_listener_set_client_func_full(listener, + socket_accept_incoming_migration, + NULL, NULL, + g_main_context_get_thread_default()); + + for (i = 0; i < listener->nsioc; i++) { + SocketAddress *address = + qio_channel_socket_get_local_address(listener->sioc[i], errp); + if (!address) { + return; + } + migrate_add_address(address); + qapi_free_SocketAddress(address); + } +} + +void socket_start_incoming_migration(const char *str, Error **errp) +{ + Error *err = NULL; + SocketAddress *saddr = socket_parse(str, &err); + if (!err) { + socket_start_incoming_migration_internal(saddr, &err); + } + qapi_free_SocketAddress(saddr); + error_propagate(errp, err); +} diff --git a/migration/socket.h b/migration/socket.h new file mode 100644 index 000000000..891dbccce --- /dev/null +++ b/migration/socket.h @@ -0,0 +1,30 @@ +/* + * QEMU live migration via socket + * + * Copyright Red Hat, Inc. 2009-2016 + * + * Authors: + * Chris Lalancette <clalance@redhat.com> + * Daniel P. Berrange <berrange@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * Contributions after 2012-01-13 are licensed under the terms of the + * GNU GPL, version 2 or (at your option) any later version. + */ + +#ifndef QEMU_MIGRATION_SOCKET_H +#define QEMU_MIGRATION_SOCKET_H + +#include "io/channel.h" +#include "io/task.h" + +void socket_send_channel_create(QIOTaskFunc f, void *data); +int socket_send_channel_destroy(QIOChannel *send); + +void socket_start_incoming_migration(const char *str, Error **errp); + +void socket_start_outgoing_migration(MigrationState *s, const char *str, + Error **errp); +#endif diff --git a/migration/target.c b/migration/target.c new file mode 100644 index 000000000..907ebf0a0 --- /dev/null +++ b/migration/target.c @@ -0,0 +1,25 @@ +/* + * QEMU live migration - functions that need to be compiled target-specific + * + * This work is licensed under the terms of the GNU GPL, version 2 + * or (at your option) any later version. + */ + +#include "qemu/osdep.h" +#include "qapi/qapi-types-migration.h" +#include "migration.h" + +#ifdef CONFIG_VFIO +#include "hw/vfio/vfio-common.h" +#endif + +void populate_vfio_info(MigrationInfo *info) +{ +#ifdef CONFIG_VFIO + if (vfio_mig_active()) { + info->has_vfio = true; + info->vfio = g_malloc0(sizeof(*info->vfio)); + info->vfio->transferred = vfio_mig_bytes_transferred(); + } +#endif +} diff --git a/migration/tls.c b/migration/tls.c new file mode 100644 index 000000000..ca1ea3bbd --- /dev/null +++ b/migration/tls.c @@ -0,0 +1,172 @@ +/* + * QEMU migration TLS support + * + * Copyright (c) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + * + */ + +#include "qemu/osdep.h" +#include "channel.h" +#include "migration.h" +#include "tls.h" +#include "crypto/tlscreds.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "trace.h" + +static QCryptoTLSCreds * +migration_tls_get_creds(MigrationState *s, + QCryptoTLSCredsEndpoint endpoint, + Error **errp) +{ + Object *creds; + QCryptoTLSCreds *ret; + + creds = object_resolve_path_component( + object_get_objects_root(), s->parameters.tls_creds); + if (!creds) { + error_setg(errp, "No TLS credentials with id '%s'", + s->parameters.tls_creds); + return NULL; + } + ret = (QCryptoTLSCreds *)object_dynamic_cast( + creds, TYPE_QCRYPTO_TLS_CREDS); + if (!ret) { + error_setg(errp, "Object with id '%s' is not TLS credentials", + s->parameters.tls_creds); + return NULL; + } + if (!qcrypto_tls_creds_check_endpoint(ret, endpoint, errp)) { + return NULL; + } + + return ret; +} + + +static void migration_tls_incoming_handshake(QIOTask *task, + gpointer opaque) +{ + QIOChannel *ioc = QIO_CHANNEL(qio_task_get_source(task)); + Error *err = NULL; + + if (qio_task_propagate_error(task, &err)) { + trace_migration_tls_incoming_handshake_error(error_get_pretty(err)); + error_report_err(err); + } else { + trace_migration_tls_incoming_handshake_complete(); + migration_channel_process_incoming(ioc); + } + object_unref(OBJECT(ioc)); +} + +void migration_tls_channel_process_incoming(MigrationState *s, + QIOChannel *ioc, + Error **errp) +{ + QCryptoTLSCreds *creds; + QIOChannelTLS *tioc; + + creds = migration_tls_get_creds( + s, QCRYPTO_TLS_CREDS_ENDPOINT_SERVER, errp); + if (!creds) { + return; + } + + tioc = qio_channel_tls_new_server( + ioc, creds, + s->parameters.tls_authz, + errp); + if (!tioc) { + return; + } + + trace_migration_tls_incoming_handshake_start(); + qio_channel_set_name(QIO_CHANNEL(tioc), "migration-tls-incoming"); + qio_channel_tls_handshake(tioc, + migration_tls_incoming_handshake, + NULL, + NULL, + NULL); +} + + +static void migration_tls_outgoing_handshake(QIOTask *task, + gpointer opaque) +{ + MigrationState *s = opaque; + QIOChannel *ioc = QIO_CHANNEL(qio_task_get_source(task)); + Error *err = NULL; + + if (qio_task_propagate_error(task, &err)) { + trace_migration_tls_outgoing_handshake_error(error_get_pretty(err)); + } else { + trace_migration_tls_outgoing_handshake_complete(); + } + migration_channel_connect(s, ioc, NULL, err); + object_unref(OBJECT(ioc)); +} + +QIOChannelTLS *migration_tls_client_create(MigrationState *s, + QIOChannel *ioc, + const char *hostname, + Error **errp) +{ + QCryptoTLSCreds *creds; + QIOChannelTLS *tioc; + + creds = migration_tls_get_creds( + s, QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT, errp); + if (!creds) { + return NULL; + } + + if (s->parameters.tls_hostname && *s->parameters.tls_hostname) { + hostname = s->parameters.tls_hostname; + } + if (!hostname) { + error_setg(errp, "No hostname available for TLS"); + return NULL; + } + + tioc = qio_channel_tls_new_client( + ioc, creds, hostname, errp); + + return tioc; +} + +void migration_tls_channel_connect(MigrationState *s, + QIOChannel *ioc, + const char *hostname, + Error **errp) +{ + QIOChannelTLS *tioc; + + tioc = migration_tls_client_create(s, ioc, hostname, errp); + if (!tioc) { + return; + } + + /* Save hostname into MigrationState for handshake */ + s->hostname = g_strdup(hostname); + trace_migration_tls_outgoing_handshake_start(hostname); + qio_channel_set_name(QIO_CHANNEL(tioc), "migration-tls-outgoing"); + qio_channel_tls_handshake(tioc, + migration_tls_outgoing_handshake, + s, + NULL, + NULL); +} diff --git a/migration/tls.h b/migration/tls.h new file mode 100644 index 000000000..de4fe2caf --- /dev/null +++ b/migration/tls.h @@ -0,0 +1,40 @@ +/* + * QEMU migration TLS support + * + * Copyright (c) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + * + */ + +#ifndef QEMU_MIGRATION_TLS_H +#define QEMU_MIGRATION_TLS_H + +#include "io/channel.h" +#include "io/channel-tls.h" + +void migration_tls_channel_process_incoming(MigrationState *s, + QIOChannel *ioc, + Error **errp); + +QIOChannelTLS *migration_tls_client_create(MigrationState *s, + QIOChannel *ioc, + const char *hostname, + Error **errp); + +void migration_tls_channel_connect(MigrationState *s, + QIOChannel *ioc, + const char *hostname, + Error **errp); +#endif diff --git a/migration/trace-events b/migration/trace-events new file mode 100644 index 000000000..b48d873b8 --- /dev/null +++ b/migration/trace-events @@ -0,0 +1,350 @@ +# See docs/devel/tracing.rst for syntax documentation. + +# savevm.c +qemu_loadvm_state_section(unsigned int section_type) "%d" +qemu_loadvm_state_section_command(int ret) "%d" +qemu_loadvm_state_section_partend(uint32_t section_id) "%u" +qemu_loadvm_state_post_main(int ret) "%d" +qemu_loadvm_state_section_startfull(uint32_t section_id, const char *idstr, uint32_t instance_id, uint32_t version_id) "%u(%s) %u %u" +qemu_savevm_send_packaged(void) "" +loadvm_state_setup(void) "" +loadvm_state_cleanup(void) "" +loadvm_handle_cmd_packaged(unsigned int length) "%u" +loadvm_handle_cmd_packaged_main(int ret) "%d" +loadvm_handle_cmd_packaged_received(int ret) "%d" +loadvm_handle_recv_bitmap(char *s) "%s" +loadvm_postcopy_handle_advise(void) "" +loadvm_postcopy_handle_listen(void) "" +loadvm_postcopy_handle_run(void) "" +loadvm_postcopy_handle_run_cpu_sync(void) "" +loadvm_postcopy_handle_run_vmstart(void) "" +loadvm_postcopy_handle_resume(void) "" +loadvm_postcopy_ram_handle_discard(void) "" +loadvm_postcopy_ram_handle_discard_end(void) "" +loadvm_postcopy_ram_handle_discard_header(const char *ramid, uint16_t len) "%s: %ud" +loadvm_process_command(uint16_t com, uint16_t len) "com=0x%x len=%d" +loadvm_process_command_ping(uint32_t val) "0x%x" +postcopy_ram_listen_thread_exit(void) "" +postcopy_ram_listen_thread_start(void) "" +qemu_savevm_send_postcopy_advise(void) "" +qemu_savevm_send_postcopy_ram_discard(const char *id, uint16_t len) "%s: %ud" +savevm_command_send(uint16_t command, uint16_t len) "com=0x%x len=%d" +savevm_section_start(const char *id, unsigned int section_id) "%s, section_id %u" +savevm_section_end(const char *id, unsigned int section_id, int ret) "%s, section_id %u -> %d" +savevm_section_skip(const char *id, unsigned int section_id) "%s, section_id %u" +savevm_send_open_return_path(void) "" +savevm_send_ping(uint32_t val) "0x%x" +savevm_send_postcopy_listen(void) "" +savevm_send_postcopy_run(void) "" +savevm_send_postcopy_resume(void) "" +savevm_send_colo_enable(void) "" +savevm_send_recv_bitmap(char *name) "%s" +savevm_state_setup(void) "" +savevm_state_resume_prepare(void) "" +savevm_state_header(void) "" +savevm_state_iterate(void) "" +savevm_state_cleanup(void) "" +savevm_state_complete_precopy(void) "" +vmstate_save(const char *idstr, const char *vmsd_name) "%s, %s" +vmstate_load(const char *idstr, const char *vmsd_name) "%s, %s" +postcopy_pause_incoming(void) "" +postcopy_pause_incoming_continued(void) "" +postcopy_page_req_sync(void *host_addr) "sync page req %p" + +# vmstate.c +vmstate_load_field_error(const char *field, int ret) "field \"%s\" load failed, ret = %d" +vmstate_load_state(const char *name, int version_id) "%s v%d" +vmstate_load_state_end(const char *name, const char *reason, int val) "%s %s/%d" +vmstate_load_state_field(const char *name, const char *field) "%s:%s" +vmstate_n_elems(const char *name, int n_elems) "%s: %d" +vmstate_subsection_load(const char *parent) "%s" +vmstate_subsection_load_bad(const char *parent, const char *sub, const char *sub2) "%s: %s/%s" +vmstate_subsection_load_good(const char *parent) "%s" +vmstate_save_state_pre_save_res(const char *name, int res) "%s/%d" +vmstate_save_state_loop(const char *name, const char *field, int n_elems) "%s/%s[%d]" +vmstate_save_state_top(const char *idstr) "%s" +vmstate_subsection_save_loop(const char *name, const char *sub) "%s/%s" +vmstate_subsection_save_top(const char *idstr) "%s" + +# vmstate-types.c +get_qtailq(const char *name, int version_id) "%s v%d" +get_qtailq_end(const char *name, const char *reason, int val) "%s %s/%d" +put_qtailq(const char *name, int version_id) "%s v%d" +put_qtailq_end(const char *name, const char *reason) "%s %s" + +get_gtree(const char *field_name, const char *key_vmsd_name, const char *val_vmsd_name, uint32_t nnodes) "%s(%s/%s) nnodes=%d" +get_gtree_end(const char *field_name, const char *key_vmsd_name, const char *val_vmsd_name, int ret) "%s(%s/%s) %d" +put_gtree(const char *field_name, const char *key_vmsd_name, const char *val_vmsd_name, uint32_t nnodes) "%s(%s/%s) nnodes=%d" +put_gtree_end(const char *field_name, const char *key_vmsd_name, const char *val_vmsd_name, int ret) "%s(%s/%s) %d" + +get_qlist(const char *field_name, const char *vmsd_name, int version_id) "%s(%s v%d)" +get_qlist_end(const char *field_name, const char *vmsd_name) "%s(%s)" +put_qlist(const char *field_name, const char *vmsd_name, int version_id) "%s(%s v%d)" +put_qlist_end(const char *field_name, const char *vmsd_name) "%s(%s)" + +# qemu-file.c +qemu_file_fclose(void) "" + +# ram.c +get_queued_page(const char *block_name, uint64_t tmp_offset, unsigned long page_abs) "%s/0x%" PRIx64 " page_abs=0x%lx" +get_queued_page_not_dirty(const char *block_name, uint64_t tmp_offset, unsigned long page_abs) "%s/0x%" PRIx64 " page_abs=0x%lx" +migration_bitmap_sync_start(void) "" +migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64 +migration_bitmap_clear_dirty(char *str, uint64_t start, uint64_t size, unsigned long page) "rb %s start 0x%"PRIx64" size 0x%"PRIx64" page 0x%lx" +migration_throttle(void) "" +ram_discard_range(const char *rbname, uint64_t start, size_t len) "%s: start: %" PRIx64 " %zx" +ram_load_loop(const char *rbname, uint64_t addr, int flags, void *host) "%s: addr: 0x%" PRIx64 " flags: 0x%x host: %p" +ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x" +ram_postcopy_send_discard_bitmap(void) "" +ram_save_page(const char *rbname, uint64_t offset, void *host) "%s: offset: 0x%" PRIx64 " host: %p" +ram_save_queue_pages(const char *rbname, size_t start, size_t len) "%s: start: 0x%zx len: 0x%zx" +ram_dirty_bitmap_request(char *str) "%s" +ram_dirty_bitmap_reload_begin(char *str) "%s" +ram_dirty_bitmap_reload_complete(char *str) "%s" +ram_dirty_bitmap_sync_start(void) "" +ram_dirty_bitmap_sync_wait(void) "" +ram_dirty_bitmap_sync_complete(void) "" +ram_state_resume_prepare(uint64_t v) "%" PRId64 +colo_flush_ram_cache_begin(uint64_t dirty_pages) "dirty_pages %" PRIu64 +colo_flush_ram_cache_end(void) "" +save_xbzrle_page_skipping(void) "" +save_xbzrle_page_overflow(void) "" +ram_save_iterate_big_wait(uint64_t milliconds, int iterations) "big wait: %" PRIu64 " milliseconds, %d iterations" +ram_load_complete(int ret, uint64_t seq_iter) "exit_code %d seq iteration %" PRIu64 +ram_write_tracking_ramblock_start(const char *block_id, size_t page_size, void *addr, size_t length) "%s: page_size: %zu addr: %p length: %zu" +ram_write_tracking_ramblock_stop(const char *block_id, size_t page_size, void *addr, size_t length) "%s: page_size: %zu addr: %p length: %zu" + +# multifd.c +multifd_new_send_channel_async(uint8_t id) "channel %d" +multifd_recv(uint8_t id, uint64_t packet_num, uint32_t used, uint32_t flags, uint32_t next_packet_size) "channel %d packet_num %" PRIu64 " pages %d flags 0x%x next packet size %d" +multifd_recv_new_channel(uint8_t id) "channel %d" +multifd_recv_sync_main(long packet_num) "packet num %ld" +multifd_recv_sync_main_signal(uint8_t id) "channel %d" +multifd_recv_sync_main_wait(uint8_t id) "channel %d" +multifd_recv_terminate_threads(bool error) "error %d" +multifd_recv_thread_end(uint8_t id, uint64_t packets, uint64_t pages) "channel %d packets %" PRIu64 " pages %" PRIu64 +multifd_recv_thread_start(uint8_t id) "%d" +multifd_send(uint8_t id, uint64_t packet_num, uint32_t used, uint32_t flags, uint32_t next_packet_size) "channel %d packet_num %" PRIu64 " pages %d flags 0x%x next packet size %d" +multifd_send_error(uint8_t id) "channel %d" +multifd_send_sync_main(long packet_num) "packet num %ld" +multifd_send_sync_main_signal(uint8_t id) "channel %d" +multifd_send_sync_main_wait(uint8_t id) "channel %d" +multifd_send_terminate_threads(bool error) "error %d" +multifd_send_thread_end(uint8_t id, uint64_t packets, uint64_t pages) "channel %d packets %" PRIu64 " pages %" PRIu64 +multifd_send_thread_start(uint8_t id) "%d" +multifd_tls_outgoing_handshake_start(void *ioc, void *tioc, const char *hostname) "ioc=%p tioc=%p hostname=%s" +multifd_tls_outgoing_handshake_error(void *ioc, const char *err) "ioc=%p err=%s" +multifd_tls_outgoing_handshake_complete(void *ioc) "ioc=%p" +multifd_set_outgoing_channel(void *ioc, const char *ioctype, const char *hostname, void *err) "ioc=%p ioctype=%s hostname=%s err=%p" + +# migration.c +await_return_path_close_on_source_close(void) "" +await_return_path_close_on_source_joining(void) "" +migrate_set_state(const char *new_state) "new state %s" +migrate_fd_cleanup(void) "" +migrate_fd_error(const char *error_desc) "error=%s" +migrate_fd_cancel(void) "" +migrate_handle_rp_req_pages(const char *rbname, size_t start, size_t len) "in %s at 0x%zx len 0x%zx" +migrate_pending(uint64_t size, uint64_t max, uint64_t pre, uint64_t compat, uint64_t post) "pending size %" PRIu64 " max %" PRIu64 " (pre = %" PRIu64 " compat=%" PRIu64 " post=%" PRIu64 ")" +migrate_send_rp_message(int msg_type, uint16_t len) "%d: len %d" +migrate_send_rp_recv_bitmap(char *name, int64_t size) "block '%s' size 0x%"PRIi64 +migration_completion_file_err(void) "" +migration_completion_vm_stop(int ret) "ret %d" +migration_completion_postcopy_end(void) "" +migration_completion_postcopy_end_after_complete(void) "" +migration_rate_limit_pre(int ms) "%d ms" +migration_rate_limit_post(int urgent) "urgent: %d" +migration_return_path_end_before(void) "" +migration_return_path_end_after(int rp_error) "%d" +migration_thread_after_loop(void) "" +migration_thread_file_err(void) "" +migration_thread_setup_complete(void) "" +open_return_path_on_source(void) "" +open_return_path_on_source_continue(void) "" +postcopy_start(void) "" +postcopy_pause_return_path(void) "" +postcopy_pause_return_path_continued(void) "" +postcopy_pause_continued(void) "" +postcopy_start_set_run(void) "" +postcopy_page_req_add(void *addr, int count) "new page req %p total %d" +source_return_path_thread_bad_end(void) "" +source_return_path_thread_end(void) "" +source_return_path_thread_entry(void) "" +source_return_path_thread_loop_top(void) "" +source_return_path_thread_pong(uint32_t val) "0x%x" +source_return_path_thread_shut(uint32_t val) "0x%x" +source_return_path_thread_resume_ack(uint32_t v) "%"PRIu32 +migration_thread_low_pending(uint64_t pending) "%" PRIu64 +migrate_transferred(uint64_t tranferred, uint64_t time_spent, uint64_t bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %" PRIu64 " max_size %" PRId64 +process_incoming_migration_co_end(int ret, int ps) "ret=%d postcopy-state=%d" +process_incoming_migration_co_postcopy_end_main(void) "" + +# channel.c +migration_set_incoming_channel(void *ioc, const char *ioctype) "ioc=%p ioctype=%s" +migration_set_outgoing_channel(void *ioc, const char *ioctype, const char *hostname, void *err) "ioc=%p ioctype=%s hostname=%s err=%p" + +# global_state.c +migrate_state_too_big(void) "" +migrate_global_state_post_load(const char *state) "loaded state: %s" +migrate_global_state_pre_save(const char *state) "saved state: %s" + +# rdma.c +qemu_rdma_accept_incoming_migration(void) "" +qemu_rdma_accept_incoming_migration_accepted(void) "" +qemu_rdma_accept_pin_state(bool pin) "%d" +qemu_rdma_accept_pin_verbsc(void *verbs) "Verbs context after listen: %p" +qemu_rdma_block_for_wrid_miss(const char *wcompstr, int wcomp, const char *gcompstr, uint64_t req) "A Wanted wrid %s (%d) but got %s (%" PRIu64 ")" +qemu_rdma_cleanup_disconnect(void) "" +qemu_rdma_close(void) "" +qemu_rdma_connect_pin_all_requested(void) "" +qemu_rdma_connect_pin_all_outcome(bool pin) "%d" +qemu_rdma_dest_init_trying(const char *host, const char *ip) "%s => %s" +qemu_rdma_dump_gid(const char *who, const char *src, const char *dst) "%s Source GID: %s, Dest GID: %s" +qemu_rdma_exchange_get_response_start(const char *desc) "CONTROL: %s receiving..." +qemu_rdma_exchange_get_response_none(const char *desc, int type) "Surprise: got %s (%d)" +qemu_rdma_exchange_send_issue_callback(void) "" +qemu_rdma_exchange_send_waiting(const char *desc) "Waiting for response %s" +qemu_rdma_exchange_send_received(const char *desc) "Response %s received." +qemu_rdma_fill(size_t control_len, size_t size) "RDMA %zd of %zd bytes already in buffer" +qemu_rdma_init_ram_blocks(int blocks) "Allocated %d local ram block structures" +qemu_rdma_poll_recv(const char *compstr, int64_t comp, int64_t id, int sent) "completion %s #%" PRId64 " received (%" PRId64 ") left %d" +qemu_rdma_poll_write(const char *compstr, int64_t comp, int left, uint64_t block, uint64_t chunk, void *local, void *remote) "completions %s (%" PRId64 ") left %d, block %" PRIu64 ", chunk: %" PRIu64 " %p %p" +qemu_rdma_poll_other(const char *compstr, int64_t comp, int left) "other completion %s (%" PRId64 ") received left %d" +qemu_rdma_post_send_control(const char *desc) "CONTROL: sending %s.." +qemu_rdma_register_and_get_keys(uint64_t len, void *start) "Registering %" PRIu64 " bytes @ %p" +qemu_rdma_register_odp_mr(const char *name) "Try to register On-Demand Paging memory region: %s" +qemu_rdma_advise_mr(const char *name, uint32_t len, uint64_t addr, const char *res) "Try to advise block %s prefetch at %" PRIu32 "@0x%" PRIx64 ": %s" +qemu_rdma_registration_handle_compress(int64_t length, int index, int64_t offset) "Zapping zero chunk: %" PRId64 " bytes, index %d, offset %" PRId64 +qemu_rdma_registration_handle_finished(void) "" +qemu_rdma_registration_handle_ram_blocks(void) "" +qemu_rdma_registration_handle_ram_blocks_loop(const char *name, uint64_t offset, uint64_t length, void *local_host_addr, unsigned int src_index) "%s: @0x%" PRIx64 "/%" PRIu64 " host:@%p src_index: %u" +qemu_rdma_registration_handle_register(int requests) "%d requests" +qemu_rdma_registration_handle_register_loop(int req, int index, uint64_t addr, uint64_t chunks) "Registration request (%d): index %d, current_addr %" PRIu64 " chunks: %" PRIu64 +qemu_rdma_registration_handle_register_rkey(int rkey) "0x%x" +qemu_rdma_registration_handle_unregister(int requests) "%d requests" +qemu_rdma_registration_handle_unregister_loop(int count, int index, uint64_t chunk) "Unregistration request (%d): index %d, chunk %" PRIu64 +qemu_rdma_registration_handle_unregister_success(uint64_t chunk) "%" PRIu64 +qemu_rdma_registration_handle_wait(void) "" +qemu_rdma_registration_start(uint64_t flags) "%" PRIu64 +qemu_rdma_registration_stop(uint64_t flags) "%" PRIu64 +qemu_rdma_registration_stop_ram(void) "" +qemu_rdma_resolve_host_trying(const char *host, const char *ip) "Trying %s => %s" +qemu_rdma_signal_unregister_append(uint64_t chunk, int pos) "Appending unregister chunk %" PRIu64 " at position %d" +qemu_rdma_signal_unregister_already(uint64_t chunk) "Unregister chunk %" PRIu64 " already in queue" +qemu_rdma_unregister_waiting_inflight(uint64_t chunk) "Cannot unregister inflight chunk: %" PRIu64 +qemu_rdma_unregister_waiting_proc(uint64_t chunk, int pos) "Processing unregister for chunk: %" PRIu64 " at position %d" +qemu_rdma_unregister_waiting_send(uint64_t chunk) "Sending unregister for chunk: %" PRIu64 +qemu_rdma_unregister_waiting_complete(uint64_t chunk) "Unregister for chunk: %" PRIu64 " complete." +qemu_rdma_write_flush(int sent) "sent total: %d" +qemu_rdma_write_one_block(int count, int block, uint64_t chunk, uint64_t current, uint64_t len, int nb_sent, int nb_chunks) "(%d) Not clobbering: block: %d chunk %" PRIu64 " current %" PRIu64 " len %" PRIu64 " %d %d" +qemu_rdma_write_one_post(uint64_t chunk, long addr, long remote, uint32_t len) "Posting chunk: %" PRIu64 ", addr: 0x%lx remote: 0x%lx, bytes %" PRIu32 +qemu_rdma_write_one_queue_full(void) "" +qemu_rdma_write_one_recvregres(int mykey, int theirkey, uint64_t chunk) "Received registration result: my key: 0x%x their key 0x%x, chunk %" PRIu64 +qemu_rdma_write_one_sendreg(uint64_t chunk, int len, int index, int64_t offset) "Sending registration request chunk %" PRIu64 " for %d bytes, index: %d, offset: %" PRId64 +qemu_rdma_write_one_top(uint64_t chunks, uint64_t size) "Writing %" PRIu64 " chunks, (%" PRIu64 " MB)" +qemu_rdma_write_one_zero(uint64_t chunk, int len, int index, int64_t offset) "Entire chunk is zero, sending compress: %" PRIu64 " for %d bytes, index: %d, offset: %" PRId64 +rdma_add_block(const char *block_name, int block, uint64_t addr, uint64_t offset, uint64_t len, uint64_t end, uint64_t bits, int chunks) "Added Block: '%s':%d, addr: %" PRIu64 ", offset: %" PRIu64 " length: %" PRIu64 " end: %" PRIu64 " bits %" PRIu64 " chunks %d" +rdma_block_notification_handle(const char *name, int index) "%s at %d" +rdma_delete_block(void *block, uint64_t addr, uint64_t offset, uint64_t len, uint64_t end, uint64_t bits, int chunks) "Deleted Block: %p, addr: %" PRIu64 ", offset: %" PRIu64 " length: %" PRIu64 " end: %" PRIu64 " bits %" PRIu64 " chunks %d" +rdma_start_incoming_migration(void) "" +rdma_start_incoming_migration_after_dest_init(void) "" +rdma_start_incoming_migration_after_rdma_listen(void) "" +rdma_start_outgoing_migration_after_rdma_connect(void) "" +rdma_start_outgoing_migration_after_rdma_source_init(void) "" + +# postcopy-ram.c +postcopy_discard_send_finish(const char *ramblock, int nwords, int ncmds) "%s mask words sent=%d in %d commands" +postcopy_discard_send_range(const char *ramblock, unsigned long start, unsigned long length) "%s:%lx/%lx" +postcopy_cleanup_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=0x%zx length=0x%zx" +postcopy_init_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=0x%zx length=0x%zx" +postcopy_nhp_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=0x%zx length=0x%zx" +postcopy_place_page(void *host_addr) "host=%p" +postcopy_place_page_zero(void *host_addr) "host=%p" +postcopy_ram_enable_notify(void) "" +mark_postcopy_blocktime_begin(uint64_t addr, void *dd, uint32_t time, int cpu, int received) "addr: 0x%" PRIx64 ", dd: %p, time: %u, cpu: %d, already_received: %d" +mark_postcopy_blocktime_end(uint64_t addr, void *dd, uint32_t time, int affected_cpu) "addr: 0x%" PRIx64 ", dd: %p, time: %u, affected_cpu: %d" +postcopy_pause_fault_thread(void) "" +postcopy_pause_fault_thread_continued(void) "" +postcopy_ram_fault_thread_entry(void) "" +postcopy_ram_fault_thread_exit(void) "" +postcopy_ram_fault_thread_fds_core(int baseufd, int quitfd) "ufd: %d quitfd: %d" +postcopy_ram_fault_thread_fds_extra(size_t index, const char *name, int fd) "%zd/%s: %d" +postcopy_ram_fault_thread_quit(void) "" +postcopy_ram_fault_thread_request(uint64_t hostaddr, const char *ramblock, size_t offset, uint32_t pid) "Request for HVA=0x%" PRIx64 " rb=%s offset=0x%zx pid=%u" +postcopy_ram_incoming_cleanup_closeuf(void) "" +postcopy_ram_incoming_cleanup_entry(void) "" +postcopy_ram_incoming_cleanup_exit(void) "" +postcopy_ram_incoming_cleanup_join(void) "" +postcopy_ram_incoming_cleanup_blocktime(uint64_t total) "total blocktime %" PRIu64 +postcopy_request_shared_page(const char *sharer, const char *rb, uint64_t rb_offset) "for %s in %s offset 0x%"PRIx64 +postcopy_request_shared_page_present(const char *sharer, const char *rb, uint64_t rb_offset) "%s already %s offset 0x%"PRIx64 +postcopy_wake_shared(uint64_t client_addr, const char *rb) "at 0x%"PRIx64" in %s" +postcopy_page_req_del(void *addr, int count) "resolved page req %p total %d" + +get_mem_fault_cpu_index(int cpu, uint32_t pid) "cpu: %d, pid: %u" + +# exec.c +migration_exec_outgoing(const char *cmd) "cmd=%s" +migration_exec_incoming(const char *cmd) "cmd=%s" + +# fd.c +migration_fd_outgoing(int fd) "fd=%d" +migration_fd_incoming(int fd) "fd=%d" + +# socket.c +migration_socket_incoming_accepted(void) "" +migration_socket_outgoing_connected(const char *hostname) "hostname=%s" +migration_socket_outgoing_error(const char *err) "error=%s" + +# tls.c +migration_tls_outgoing_handshake_start(const char *hostname) "hostname=%s" +migration_tls_outgoing_handshake_error(const char *err) "err=%s" +migration_tls_outgoing_handshake_complete(void) "" +migration_tls_incoming_handshake_start(void) "" +migration_tls_incoming_handshake_error(const char *err) "err=%s" +migration_tls_incoming_handshake_complete(void) "" + +# colo.c +colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'" +colo_send_message(const char *msg) "Send '%s' message" +colo_receive_message(const char *msg) "Receive '%s' message" + +# colo-failover.c +colo_failover_set_state(const char *new_state) "new state %s" + +# block-dirty-bitmap.c +send_bitmap_header_enter(void) "" +send_bitmap_bits(uint32_t flags, uint64_t start_sector, uint32_t nr_sectors, uint64_t data_size) "flags: 0x%x, start_sector: %" PRIu64 ", nr_sectors: %" PRIu32 ", data_size: %" PRIu64 +dirty_bitmap_save_iterate(int in_postcopy) "in postcopy: %d" +dirty_bitmap_save_complete_enter(void) "" +dirty_bitmap_save_complete_finish(void) "" +dirty_bitmap_save_pending(uint64_t pending, uint64_t max_size) "pending %" PRIu64 " max: %" PRIu64 +dirty_bitmap_load_complete(void) "" +dirty_bitmap_load_bits_enter(uint64_t first_sector, uint32_t nr_sectors) "chunk: %" PRIu64 " %" PRIu32 +dirty_bitmap_load_bits_zeroes(void) "" +dirty_bitmap_load_header(uint32_t flags) "flags 0x%x" +dirty_bitmap_load_enter(void) "" +dirty_bitmap_load_success(void) "" + +# dirtyrate.c +dirtyrate_set_state(const char *new_state) "new state %s" +query_dirty_rate_info(const char *new_state) "current state %s" +get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock name: %s, vfn: %"PRIu64 ", crc: %" PRIu32 +calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32 +skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64 +find_page_matched(const char *idstr) "ramblock %s addr or size changed" +dirtyrate_calculate(int64_t dirtyrate) "dirty rate: %" PRIi64 " MB/s" +dirtyrate_do_calculate_vcpu(int idx, uint64_t rate) "vcpu[%d]: %"PRIu64 " MB/s" + +# block.c +migration_block_init_shared(const char *blk_device_name) "Start migration for %s with shared base image" +migration_block_init_full(const char *blk_device_name) "Start full migration for %s" +migration_block_save_device_dirty(int64_t sector) "Error reading sector %" PRId64 +migration_block_flush_blks(const char *action, int submitted, int read_done, int transferred) "%s submitted %d read_done %d transferred %d" +migration_block_save(const char *mig_stage, int submitted, int transferred) "Enter save live %s submitted %d transferred %d" +migration_block_save_complete(void) "Block migration completed" +migration_block_save_pending(uint64_t pending) "Enter save live pending %" PRIu64 + +# page_cache.c +migration_pagecache_init(int64_t max_num_items) "Setting cache buckets to %" PRId64 +migration_pagecache_insert(void) "Error allocating page" diff --git a/migration/trace.h b/migration/trace.h new file mode 100644 index 000000000..e1a0f4fb7 --- /dev/null +++ b/migration/trace.h @@ -0,0 +1 @@ +#include "trace/trace-migration.h" diff --git a/migration/vmstate-types.c b/migration/vmstate-types.c new file mode 100644 index 000000000..bf4d44030 --- /dev/null +++ b/migration/vmstate-types.c @@ -0,0 +1,893 @@ +/* + * VMStateInfo's for basic typse + * + * Copyright (c) 2009-2017 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu-file.h" +#include "migration.h" +#include "migration/vmstate.h" +#include "qemu/error-report.h" +#include "qemu/queue.h" +#include "trace.h" + +/* bool */ + +static int get_bool(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + bool *v = pv; + *v = qemu_get_byte(f); + return 0; +} + +static int put_bool(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + bool *v = pv; + qemu_put_byte(f, *v); + return 0; +} + +const VMStateInfo vmstate_info_bool = { + .name = "bool", + .get = get_bool, + .put = put_bool, +}; + +/* 8 bit int */ + +static int get_int8(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int8_t *v = pv; + qemu_get_s8s(f, v); + return 0; +} + +static int put_int8(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + int8_t *v = pv; + qemu_put_s8s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_int8 = { + .name = "int8", + .get = get_int8, + .put = put_int8, +}; + +/* 16 bit int */ + +static int get_int16(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int16_t *v = pv; + qemu_get_sbe16s(f, v); + return 0; +} + +static int put_int16(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + int16_t *v = pv; + qemu_put_sbe16s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_int16 = { + .name = "int16", + .get = get_int16, + .put = put_int16, +}; + +/* 32 bit int */ + +static int get_int32(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int32_t *v = pv; + qemu_get_sbe32s(f, v); + return 0; +} + +static int put_int32(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + int32_t *v = pv; + qemu_put_sbe32s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_int32 = { + .name = "int32", + .get = get_int32, + .put = put_int32, +}; + +/* 32 bit int. See that the received value is the same than the one + in the field */ + +static int get_int32_equal(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int32_t *v = pv; + int32_t v2; + qemu_get_sbe32s(f, &v2); + + if (*v == v2) { + return 0; + } + error_report("%" PRIx32 " != %" PRIx32, *v, v2); + if (field->err_hint) { + error_printf("%s\n", field->err_hint); + } + return -EINVAL; +} + +const VMStateInfo vmstate_info_int32_equal = { + .name = "int32 equal", + .get = get_int32_equal, + .put = put_int32, +}; + +/* 32 bit int. Check that the received value is non-negative + * and less than or equal to the one in the field. + */ + +static int get_int32_le(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int32_t *cur = pv; + int32_t loaded; + qemu_get_sbe32s(f, &loaded); + + if (loaded >= 0 && loaded <= *cur) { + *cur = loaded; + return 0; + } + error_report("Invalid value %" PRId32 + " expecting positive value <= %" PRId32, + loaded, *cur); + return -EINVAL; +} + +const VMStateInfo vmstate_info_int32_le = { + .name = "int32 le", + .get = get_int32_le, + .put = put_int32, +}; + +/* 64 bit int */ + +static int get_int64(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int64_t *v = pv; + qemu_get_sbe64s(f, v); + return 0; +} + +static int put_int64(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + int64_t *v = pv; + qemu_put_sbe64s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_int64 = { + .name = "int64", + .get = get_int64, + .put = put_int64, +}; + +/* 8 bit unsigned int */ + +static int get_uint8(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint8_t *v = pv; + qemu_get_8s(f, v); + return 0; +} + +static int put_uint8(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + uint8_t *v = pv; + qemu_put_8s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_uint8 = { + .name = "uint8", + .get = get_uint8, + .put = put_uint8, +}; + +/* 16 bit unsigned int */ + +static int get_uint16(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint16_t *v = pv; + qemu_get_be16s(f, v); + return 0; +} + +static int put_uint16(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + uint16_t *v = pv; + qemu_put_be16s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_uint16 = { + .name = "uint16", + .get = get_uint16, + .put = put_uint16, +}; + +/* 32 bit unsigned int */ + +static int get_uint32(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint32_t *v = pv; + qemu_get_be32s(f, v); + return 0; +} + +static int put_uint32(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + uint32_t *v = pv; + qemu_put_be32s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_uint32 = { + .name = "uint32", + .get = get_uint32, + .put = put_uint32, +}; + +/* 32 bit uint. See that the received value is the same than the one + in the field */ + +static int get_uint32_equal(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint32_t *v = pv; + uint32_t v2; + qemu_get_be32s(f, &v2); + + if (*v == v2) { + return 0; + } + error_report("%" PRIx32 " != %" PRIx32, *v, v2); + if (field->err_hint) { + error_printf("%s\n", field->err_hint); + } + return -EINVAL; +} + +const VMStateInfo vmstate_info_uint32_equal = { + .name = "uint32 equal", + .get = get_uint32_equal, + .put = put_uint32, +}; + +/* 64 bit unsigned int */ + +static int get_uint64(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint64_t *v = pv; + qemu_get_be64s(f, v); + return 0; +} + +static int put_uint64(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + uint64_t *v = pv; + qemu_put_be64s(f, v); + return 0; +} + +const VMStateInfo vmstate_info_uint64 = { + .name = "uint64", + .get = get_uint64, + .put = put_uint64, +}; + +static int get_nullptr(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) + +{ + if (qemu_get_byte(f) == VMS_NULLPTR_MARKER) { + return 0; + } + error_report("vmstate: get_nullptr expected VMS_NULLPTR_MARKER"); + return -EINVAL; +} + +static int put_nullptr(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) + +{ + if (pv == NULL) { + qemu_put_byte(f, VMS_NULLPTR_MARKER); + return 0; + } + error_report("vmstate: put_nullptr must be called with pv == NULL"); + return -EINVAL; +} + +const VMStateInfo vmstate_info_nullptr = { + .name = "uint64", + .get = get_nullptr, + .put = put_nullptr, +}; + +/* 64 bit unsigned int. See that the received value is the same than the one + in the field */ + +static int get_uint64_equal(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint64_t *v = pv; + uint64_t v2; + qemu_get_be64s(f, &v2); + + if (*v == v2) { + return 0; + } + error_report("%" PRIx64 " != %" PRIx64, *v, v2); + if (field->err_hint) { + error_printf("%s\n", field->err_hint); + } + return -EINVAL; +} + +const VMStateInfo vmstate_info_uint64_equal = { + .name = "int64 equal", + .get = get_uint64_equal, + .put = put_uint64, +}; + +/* 8 bit int. See that the received value is the same than the one + in the field */ + +static int get_uint8_equal(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint8_t *v = pv; + uint8_t v2; + qemu_get_8s(f, &v2); + + if (*v == v2) { + return 0; + } + error_report("%x != %x", *v, v2); + if (field->err_hint) { + error_printf("%s\n", field->err_hint); + } + return -EINVAL; +} + +const VMStateInfo vmstate_info_uint8_equal = { + .name = "uint8 equal", + .get = get_uint8_equal, + .put = put_uint8, +}; + +/* 16 bit unsigned int int. See that the received value is the same than the one + in the field */ + +static int get_uint16_equal(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint16_t *v = pv; + uint16_t v2; + qemu_get_be16s(f, &v2); + + if (*v == v2) { + return 0; + } + error_report("%x != %x", *v, v2); + if (field->err_hint) { + error_printf("%s\n", field->err_hint); + } + return -EINVAL; +} + +const VMStateInfo vmstate_info_uint16_equal = { + .name = "uint16 equal", + .get = get_uint16_equal, + .put = put_uint16, +}; + +/* CPU_DoubleU type */ + +static int get_cpudouble(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + CPU_DoubleU *v = pv; + qemu_get_be32s(f, &v->l.upper); + qemu_get_be32s(f, &v->l.lower); + return 0; +} + +static int put_cpudouble(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + CPU_DoubleU *v = pv; + qemu_put_be32s(f, &v->l.upper); + qemu_put_be32s(f, &v->l.lower); + return 0; +} + +const VMStateInfo vmstate_info_cpudouble = { + .name = "CPU_Double_U", + .get = get_cpudouble, + .put = put_cpudouble, +}; + +/* uint8_t buffers */ + +static int get_buffer(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint8_t *v = pv; + qemu_get_buffer(f, v, size); + return 0; +} + +static int put_buffer(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + uint8_t *v = pv; + qemu_put_buffer(f, v, size); + return 0; +} + +const VMStateInfo vmstate_info_buffer = { + .name = "buffer", + .get = get_buffer, + .put = put_buffer, +}; + +/* unused buffers: space that was used for some fields that are + not useful anymore */ + +static int get_unused_buffer(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + uint8_t buf[1024]; + int block_len; + + while (size > 0) { + block_len = MIN(sizeof(buf), size); + size -= block_len; + qemu_get_buffer(f, buf, block_len); + } + return 0; +} + +static int put_unused_buffer(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + static const uint8_t buf[1024]; + int block_len; + + while (size > 0) { + block_len = MIN(sizeof(buf), size); + size -= block_len; + qemu_put_buffer(f, buf, block_len); + } + + return 0; +} + +const VMStateInfo vmstate_info_unused_buffer = { + .name = "unused_buffer", + .get = get_unused_buffer, + .put = put_unused_buffer, +}; + +/* vmstate_info_tmp, see VMSTATE_WITH_TMP, the idea is that we allocate + * a temporary buffer and the pre_load/pre_save methods in the child vmsd + * copy stuff from the parent into the child and do calculations to fill + * in fields that don't really exist in the parent but need to be in the + * stream. + */ +static int get_tmp(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + int ret; + const VMStateDescription *vmsd = field->vmsd; + int version_id = field->version_id; + void *tmp = g_malloc(size); + + /* Writes the parent field which is at the start of the tmp */ + *(void **)tmp = pv; + ret = vmstate_load_state(f, vmsd, tmp, version_id); + g_free(tmp); + return ret; +} + +static int put_tmp(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + const VMStateDescription *vmsd = field->vmsd; + void *tmp = g_malloc(size); + int ret; + + /* Writes the parent field which is at the start of the tmp */ + *(void **)tmp = pv; + ret = vmstate_save_state(f, vmsd, tmp, vmdesc); + g_free(tmp); + + return ret; +} + +const VMStateInfo vmstate_info_tmp = { + .name = "tmp", + .get = get_tmp, + .put = put_tmp, +}; + +/* bitmaps (as defined by bitmap.h). Note that size here is the size + * of the bitmap in bits. The on-the-wire format of a bitmap is 64 + * bit words with the bits in big endian order. The in-memory format + * is an array of 'unsigned long', which may be either 32 or 64 bits. + */ +/* This is the number of 64 bit words sent over the wire */ +#define BITS_TO_U64S(nr) DIV_ROUND_UP(nr, 64) +static int get_bitmap(QEMUFile *f, void *pv, size_t size, + const VMStateField *field) +{ + unsigned long *bmp = pv; + int i, idx = 0; + for (i = 0; i < BITS_TO_U64S(size); i++) { + uint64_t w = qemu_get_be64(f); + bmp[idx++] = w; + if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) { + bmp[idx++] = w >> 32; + } + } + return 0; +} + +static int put_bitmap(QEMUFile *f, void *pv, size_t size, + const VMStateField *field, JSONWriter *vmdesc) +{ + unsigned long *bmp = pv; + int i, idx = 0; + for (i = 0; i < BITS_TO_U64S(size); i++) { + uint64_t w = bmp[idx++]; + if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) { + w |= ((uint64_t)bmp[idx++]) << 32; + } + qemu_put_be64(f, w); + } + + return 0; +} + +const VMStateInfo vmstate_info_bitmap = { + .name = "bitmap", + .get = get_bitmap, + .put = put_bitmap, +}; + +/* get for QTAILQ + * meta data about the QTAILQ is encoded in a VMStateField structure + */ +static int get_qtailq(QEMUFile *f, void *pv, size_t unused_size, + const VMStateField *field) +{ + int ret = 0; + const VMStateDescription *vmsd = field->vmsd; + /* size of a QTAILQ element */ + size_t size = field->size; + /* offset of the QTAILQ entry in a QTAILQ element */ + size_t entry_offset = field->start; + int version_id = field->version_id; + void *elm; + + trace_get_qtailq(vmsd->name, version_id); + if (version_id > vmsd->version_id) { + error_report("%s %s", vmsd->name, "too new"); + trace_get_qtailq_end(vmsd->name, "too new", -EINVAL); + + return -EINVAL; + } + if (version_id < vmsd->minimum_version_id) { + error_report("%s %s", vmsd->name, "too old"); + trace_get_qtailq_end(vmsd->name, "too old", -EINVAL); + return -EINVAL; + } + + while (qemu_get_byte(f)) { + elm = g_malloc(size); + ret = vmstate_load_state(f, vmsd, elm, version_id); + if (ret) { + return ret; + } + QTAILQ_RAW_INSERT_TAIL(pv, elm, entry_offset); + } + + trace_get_qtailq_end(vmsd->name, "end", ret); + return ret; +} + +/* put for QTAILQ */ +static int put_qtailq(QEMUFile *f, void *pv, size_t unused_size, + const VMStateField *field, JSONWriter *vmdesc) +{ + const VMStateDescription *vmsd = field->vmsd; + /* offset of the QTAILQ entry in a QTAILQ element*/ + size_t entry_offset = field->start; + void *elm; + int ret; + + trace_put_qtailq(vmsd->name, vmsd->version_id); + + QTAILQ_RAW_FOREACH(elm, pv, entry_offset) { + qemu_put_byte(f, true); + ret = vmstate_save_state(f, vmsd, elm, vmdesc); + if (ret) { + return ret; + } + } + qemu_put_byte(f, false); + + trace_put_qtailq_end(vmsd->name, "end"); + + return 0; +} +const VMStateInfo vmstate_info_qtailq = { + .name = "qtailq", + .get = get_qtailq, + .put = put_qtailq, +}; + +struct put_gtree_data { + QEMUFile *f; + const VMStateDescription *key_vmsd; + const VMStateDescription *val_vmsd; + JSONWriter *vmdesc; + int ret; +}; + +static gboolean put_gtree_elem(gpointer key, gpointer value, gpointer data) +{ + struct put_gtree_data *capsule = (struct put_gtree_data *)data; + QEMUFile *f = capsule->f; + int ret; + + qemu_put_byte(f, true); + + /* put the key */ + if (!capsule->key_vmsd) { + qemu_put_be64(f, (uint64_t)(uintptr_t)(key)); /* direct key */ + } else { + ret = vmstate_save_state(f, capsule->key_vmsd, key, capsule->vmdesc); + if (ret) { + capsule->ret = ret; + return true; + } + } + + /* put the data */ + ret = vmstate_save_state(f, capsule->val_vmsd, value, capsule->vmdesc); + if (ret) { + capsule->ret = ret; + return true; + } + return false; +} + +static int put_gtree(QEMUFile *f, void *pv, size_t unused_size, + const VMStateField *field, JSONWriter *vmdesc) +{ + bool direct_key = (!field->start); + const VMStateDescription *key_vmsd = direct_key ? NULL : &field->vmsd[1]; + const VMStateDescription *val_vmsd = &field->vmsd[0]; + const char *key_vmsd_name = direct_key ? "direct" : key_vmsd->name; + struct put_gtree_data capsule = { + .f = f, + .key_vmsd = key_vmsd, + .val_vmsd = val_vmsd, + .vmdesc = vmdesc, + .ret = 0}; + GTree **pval = pv; + GTree *tree = *pval; + uint32_t nnodes = g_tree_nnodes(tree); + int ret; + + trace_put_gtree(field->name, key_vmsd_name, val_vmsd->name, nnodes); + qemu_put_be32(f, nnodes); + g_tree_foreach(tree, put_gtree_elem, (gpointer)&capsule); + qemu_put_byte(f, false); + ret = capsule.ret; + if (ret) { + error_report("%s : failed to save gtree (%d)", field->name, ret); + } + trace_put_gtree_end(field->name, key_vmsd_name, val_vmsd->name, ret); + return ret; +} + +static int get_gtree(QEMUFile *f, void *pv, size_t unused_size, + const VMStateField *field) +{ + bool direct_key = (!field->start); + const VMStateDescription *key_vmsd = direct_key ? NULL : &field->vmsd[1]; + const VMStateDescription *val_vmsd = &field->vmsd[0]; + const char *key_vmsd_name = direct_key ? "direct" : key_vmsd->name; + int version_id = field->version_id; + size_t key_size = field->start; + size_t val_size = field->size; + int nnodes, count = 0; + GTree **pval = pv; + GTree *tree = *pval; + void *key, *val; + int ret = 0; + + /* in case of direct key, the key vmsd can be {}, ie. check fields */ + if (!direct_key && version_id > key_vmsd->version_id) { + error_report("%s %s", key_vmsd->name, "too new"); + return -EINVAL; + } + if (!direct_key && version_id < key_vmsd->minimum_version_id) { + error_report("%s %s", key_vmsd->name, "too old"); + return -EINVAL; + } + if (version_id > val_vmsd->version_id) { + error_report("%s %s", val_vmsd->name, "too new"); + return -EINVAL; + } + if (version_id < val_vmsd->minimum_version_id) { + error_report("%s %s", val_vmsd->name, "too old"); + return -EINVAL; + } + + nnodes = qemu_get_be32(f); + trace_get_gtree(field->name, key_vmsd_name, val_vmsd->name, nnodes); + + while (qemu_get_byte(f)) { + if ((++count) > nnodes) { + ret = -EINVAL; + break; + } + if (direct_key) { + key = (void *)(uintptr_t)qemu_get_be64(f); + } else { + key = g_malloc0(key_size); + ret = vmstate_load_state(f, key_vmsd, key, version_id); + if (ret) { + error_report("%s : failed to load %s (%d)", + field->name, key_vmsd->name, ret); + goto key_error; + } + } + val = g_malloc0(val_size); + ret = vmstate_load_state(f, val_vmsd, val, version_id); + if (ret) { + error_report("%s : failed to load %s (%d)", + field->name, val_vmsd->name, ret); + goto val_error; + } + g_tree_insert(tree, key, val); + } + if (count != nnodes) { + error_report("%s inconsistent stream when loading the gtree", + field->name); + return -EINVAL; + } + trace_get_gtree_end(field->name, key_vmsd_name, val_vmsd->name, ret); + return ret; +val_error: + g_free(val); +key_error: + if (!direct_key) { + g_free(key); + } + trace_get_gtree_end(field->name, key_vmsd_name, val_vmsd->name, ret); + return ret; +} + + +const VMStateInfo vmstate_info_gtree = { + .name = "gtree", + .get = get_gtree, + .put = put_gtree, +}; + +static int put_qlist(QEMUFile *f, void *pv, size_t unused_size, + const VMStateField *field, JSONWriter *vmdesc) +{ + const VMStateDescription *vmsd = field->vmsd; + /* offset of the QTAILQ entry in a QTAILQ element*/ + size_t entry_offset = field->start; + void *elm; + int ret; + + trace_put_qlist(field->name, vmsd->name, vmsd->version_id); + QLIST_RAW_FOREACH(elm, pv, entry_offset) { + qemu_put_byte(f, true); + ret = vmstate_save_state(f, vmsd, elm, vmdesc); + if (ret) { + error_report("%s: failed to save %s (%d)", field->name, + vmsd->name, ret); + return ret; + } + } + qemu_put_byte(f, false); + trace_put_qlist_end(field->name, vmsd->name); + + return 0; +} + +static int get_qlist(QEMUFile *f, void *pv, size_t unused_size, + const VMStateField *field) +{ + int ret = 0; + const VMStateDescription *vmsd = field->vmsd; + /* size of a QLIST element */ + size_t size = field->size; + /* offset of the QLIST entry in a QLIST element */ + size_t entry_offset = field->start; + int version_id = field->version_id; + void *elm, *prev = NULL; + + trace_get_qlist(field->name, vmsd->name, vmsd->version_id); + if (version_id > vmsd->version_id) { + error_report("%s %s", vmsd->name, "too new"); + return -EINVAL; + } + if (version_id < vmsd->minimum_version_id) { + error_report("%s %s", vmsd->name, "too old"); + return -EINVAL; + } + + while (qemu_get_byte(f)) { + elm = g_malloc(size); + ret = vmstate_load_state(f, vmsd, elm, version_id); + if (ret) { + error_report("%s: failed to load %s (%d)", field->name, + vmsd->name, ret); + g_free(elm); + return ret; + } + if (!prev) { + QLIST_RAW_INSERT_HEAD(pv, elm, entry_offset); + } else { + QLIST_RAW_INSERT_AFTER(pv, prev, elm, entry_offset); + } + prev = elm; + } + trace_get_qlist_end(field->name, vmsd->name); + + return ret; +} + +const VMStateInfo vmstate_info_qlist = { + .name = "qlist", + .get = get_qlist, + .put = put_qlist, +}; diff --git a/migration/vmstate.c b/migration/vmstate.c new file mode 100644 index 000000000..05f87cddd --- /dev/null +++ b/migration/vmstate.c @@ -0,0 +1,541 @@ +/* + * VMState interpreter + * + * Copyright (c) 2009-2017 Red Hat Inc + * + * Authors: + * Juan Quintela <quintela@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "migration.h" +#include "migration/vmstate.h" +#include "savevm.h" +#include "qapi/qmp/json-writer.h" +#include "qemu-file.h" +#include "qemu/bitops.h" +#include "qemu/error-report.h" +#include "trace.h" + +static int vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque, JSONWriter *vmdesc); +static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque); + +static int vmstate_n_elems(void *opaque, const VMStateField *field) +{ + int n_elems = 1; + + if (field->flags & VMS_ARRAY) { + n_elems = field->num; + } else if (field->flags & VMS_VARRAY_INT32) { + n_elems = *(int32_t *)(opaque + field->num_offset); + } else if (field->flags & VMS_VARRAY_UINT32) { + n_elems = *(uint32_t *)(opaque + field->num_offset); + } else if (field->flags & VMS_VARRAY_UINT16) { + n_elems = *(uint16_t *)(opaque + field->num_offset); + } else if (field->flags & VMS_VARRAY_UINT8) { + n_elems = *(uint8_t *)(opaque + field->num_offset); + } + + if (field->flags & VMS_MULTIPLY_ELEMENTS) { + n_elems *= field->num; + } + + trace_vmstate_n_elems(field->name, n_elems); + return n_elems; +} + +static int vmstate_size(void *opaque, const VMStateField *field) +{ + int size = field->size; + + if (field->flags & VMS_VBUFFER) { + size = *(int32_t *)(opaque + field->size_offset); + if (field->flags & VMS_MULTIPLY) { + size *= field->size; + } + } + + return size; +} + +static void vmstate_handle_alloc(void *ptr, const VMStateField *field, + void *opaque) +{ + if (field->flags & VMS_POINTER && field->flags & VMS_ALLOC) { + gsize size = vmstate_size(opaque, field); + size *= vmstate_n_elems(opaque, field); + if (size) { + *(void **)ptr = g_malloc(size); + } + } +} + +int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque, int version_id) +{ + const VMStateField *field = vmsd->fields; + int ret = 0; + + trace_vmstate_load_state(vmsd->name, version_id); + if (version_id > vmsd->version_id) { + error_report("%s: incoming version_id %d is too new " + "for local version_id %d", + vmsd->name, version_id, vmsd->version_id); + trace_vmstate_load_state_end(vmsd->name, "too new", -EINVAL); + return -EINVAL; + } + if (version_id < vmsd->minimum_version_id) { + if (vmsd->load_state_old && + version_id >= vmsd->minimum_version_id_old) { + ret = vmsd->load_state_old(f, opaque, version_id); + trace_vmstate_load_state_end(vmsd->name, "old path", ret); + return ret; + } + error_report("%s: incoming version_id %d is too old " + "for local minimum version_id %d", + vmsd->name, version_id, vmsd->minimum_version_id); + trace_vmstate_load_state_end(vmsd->name, "too old", -EINVAL); + return -EINVAL; + } + if (vmsd->pre_load) { + int ret = vmsd->pre_load(opaque); + if (ret) { + return ret; + } + } + while (field->name) { + trace_vmstate_load_state_field(vmsd->name, field->name); + if ((field->field_exists && + field->field_exists(opaque, version_id)) || + (!field->field_exists && + field->version_id <= version_id)) { + void *first_elem = opaque + field->offset; + int i, n_elems = vmstate_n_elems(opaque, field); + int size = vmstate_size(opaque, field); + + vmstate_handle_alloc(first_elem, field, opaque); + if (field->flags & VMS_POINTER) { + first_elem = *(void **)first_elem; + assert(first_elem || !n_elems || !size); + } + for (i = 0; i < n_elems; i++) { + void *curr_elem = first_elem + size * i; + + if (field->flags & VMS_ARRAY_OF_POINTER) { + curr_elem = *(void **)curr_elem; + } + if (!curr_elem && size) { + /* if null pointer check placeholder and do not follow */ + assert(field->flags & VMS_ARRAY_OF_POINTER); + ret = vmstate_info_nullptr.get(f, curr_elem, size, NULL); + } else if (field->flags & VMS_STRUCT) { + ret = vmstate_load_state(f, field->vmsd, curr_elem, + field->vmsd->version_id); + } else if (field->flags & VMS_VSTRUCT) { + ret = vmstate_load_state(f, field->vmsd, curr_elem, + field->struct_version_id); + } else { + ret = field->info->get(f, curr_elem, size, field); + } + if (ret >= 0) { + ret = qemu_file_get_error(f); + } + if (ret < 0) { + qemu_file_set_error(f, ret); + error_report("Failed to load %s:%s", vmsd->name, + field->name); + trace_vmstate_load_field_error(field->name, ret); + return ret; + } + } + } else if (field->flags & VMS_MUST_EXIST) { + error_report("Input validation failed: %s/%s", + vmsd->name, field->name); + return -1; + } + field++; + } + ret = vmstate_subsection_load(f, vmsd, opaque); + if (ret != 0) { + return ret; + } + if (vmsd->post_load) { + ret = vmsd->post_load(opaque, version_id); + } + trace_vmstate_load_state_end(vmsd->name, "end", ret); + return ret; +} + +static int vmfield_name_num(const VMStateField *start, + const VMStateField *search) +{ + const VMStateField *field; + int found = 0; + + for (field = start; field->name; field++) { + if (!strcmp(field->name, search->name)) { + if (field == search) { + return found; + } + found++; + } + } + + return -1; +} + +static bool vmfield_name_is_unique(const VMStateField *start, + const VMStateField *search) +{ + const VMStateField *field; + int found = 0; + + for (field = start; field->name; field++) { + if (!strcmp(field->name, search->name)) { + found++; + /* name found more than once, so it's not unique */ + if (found > 1) { + return false; + } + } + } + + return true; +} + +static const char *vmfield_get_type_name(const VMStateField *field) +{ + const char *type = "unknown"; + + if (field->flags & VMS_STRUCT) { + type = "struct"; + } else if (field->flags & VMS_VSTRUCT) { + type = "vstruct"; + } else if (field->info->name) { + type = field->info->name; + } + + return type; +} + +static bool vmsd_can_compress(const VMStateField *field) +{ + if (field->field_exists) { + /* Dynamically existing fields mess up compression */ + return false; + } + + if (field->flags & VMS_STRUCT) { + const VMStateField *sfield = field->vmsd->fields; + while (sfield->name) { + if (!vmsd_can_compress(sfield)) { + /* Child elements can't compress, so can't we */ + return false; + } + sfield++; + } + + if (field->vmsd->subsections) { + /* Subsections may come and go, better don't compress */ + return false; + } + } + + return true; +} + +static void vmsd_desc_field_start(const VMStateDescription *vmsd, + JSONWriter *vmdesc, + const VMStateField *field, int i, int max) +{ + char *name, *old_name; + bool is_array = max > 1; + bool can_compress = vmsd_can_compress(field); + + if (!vmdesc) { + return; + } + + name = g_strdup(field->name); + + /* Field name is not unique, need to make it unique */ + if (!vmfield_name_is_unique(vmsd->fields, field)) { + int num = vmfield_name_num(vmsd->fields, field); + old_name = name; + name = g_strdup_printf("%s[%d]", name, num); + g_free(old_name); + } + + json_writer_start_object(vmdesc, NULL); + json_writer_str(vmdesc, "name", name); + if (is_array) { + if (can_compress) { + json_writer_int64(vmdesc, "array_len", max); + } else { + json_writer_int64(vmdesc, "index", i); + } + } + json_writer_str(vmdesc, "type", vmfield_get_type_name(field)); + + if (field->flags & VMS_STRUCT) { + json_writer_start_object(vmdesc, "struct"); + } + + g_free(name); +} + +static void vmsd_desc_field_end(const VMStateDescription *vmsd, + JSONWriter *vmdesc, + const VMStateField *field, size_t size, int i) +{ + if (!vmdesc) { + return; + } + + if (field->flags & VMS_STRUCT) { + /* We printed a struct in between, close its child object */ + json_writer_end_object(vmdesc); + } + + json_writer_int64(vmdesc, "size", size); + json_writer_end_object(vmdesc); +} + + +bool vmstate_save_needed(const VMStateDescription *vmsd, void *opaque) +{ + if (vmsd->needed && !vmsd->needed(opaque)) { + /* optional section not needed */ + return false; + } + return true; +} + + +int vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque, JSONWriter *vmdesc_id) +{ + return vmstate_save_state_v(f, vmsd, opaque, vmdesc_id, vmsd->version_id); +} + +int vmstate_save_state_v(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque, JSONWriter *vmdesc, int version_id) +{ + int ret = 0; + const VMStateField *field = vmsd->fields; + + trace_vmstate_save_state_top(vmsd->name); + + if (vmsd->pre_save) { + ret = vmsd->pre_save(opaque); + trace_vmstate_save_state_pre_save_res(vmsd->name, ret); + if (ret) { + error_report("pre-save failed: %s", vmsd->name); + return ret; + } + } + + if (vmdesc) { + json_writer_str(vmdesc, "vmsd_name", vmsd->name); + json_writer_int64(vmdesc, "version", version_id); + json_writer_start_array(vmdesc, "fields"); + } + + while (field->name) { + if ((field->field_exists && + field->field_exists(opaque, version_id)) || + (!field->field_exists && + field->version_id <= version_id)) { + void *first_elem = opaque + field->offset; + int i, n_elems = vmstate_n_elems(opaque, field); + int size = vmstate_size(opaque, field); + int64_t old_offset, written_bytes; + JSONWriter *vmdesc_loop = vmdesc; + + trace_vmstate_save_state_loop(vmsd->name, field->name, n_elems); + if (field->flags & VMS_POINTER) { + first_elem = *(void **)first_elem; + assert(first_elem || !n_elems || !size); + } + for (i = 0; i < n_elems; i++) { + void *curr_elem = first_elem + size * i; + + vmsd_desc_field_start(vmsd, vmdesc_loop, field, i, n_elems); + old_offset = qemu_ftell_fast(f); + if (field->flags & VMS_ARRAY_OF_POINTER) { + assert(curr_elem); + curr_elem = *(void **)curr_elem; + } + if (!curr_elem && size) { + /* if null pointer write placeholder and do not follow */ + assert(field->flags & VMS_ARRAY_OF_POINTER); + ret = vmstate_info_nullptr.put(f, curr_elem, size, NULL, + NULL); + } else if (field->flags & VMS_STRUCT) { + ret = vmstate_save_state(f, field->vmsd, curr_elem, + vmdesc_loop); + } else if (field->flags & VMS_VSTRUCT) { + ret = vmstate_save_state_v(f, field->vmsd, curr_elem, + vmdesc_loop, + field->struct_version_id); + } else { + ret = field->info->put(f, curr_elem, size, field, + vmdesc_loop); + } + if (ret) { + error_report("Save of field %s/%s failed", + vmsd->name, field->name); + if (vmsd->post_save) { + vmsd->post_save(opaque); + } + return ret; + } + + written_bytes = qemu_ftell_fast(f) - old_offset; + vmsd_desc_field_end(vmsd, vmdesc_loop, field, written_bytes, i); + + /* Compressed arrays only care about the first element */ + if (vmdesc_loop && vmsd_can_compress(field)) { + vmdesc_loop = NULL; + } + } + } else { + if (field->flags & VMS_MUST_EXIST) { + error_report("Output state validation failed: %s/%s", + vmsd->name, field->name); + assert(!(field->flags & VMS_MUST_EXIST)); + } + } + field++; + } + + if (vmdesc) { + json_writer_end_array(vmdesc); + } + + ret = vmstate_subsection_save(f, vmsd, opaque, vmdesc); + + if (vmsd->post_save) { + int ps_ret = vmsd->post_save(opaque); + if (!ret) { + ret = ps_ret; + } + } + return ret; +} + +static const VMStateDescription * +vmstate_get_subsection(const VMStateDescription **sub, char *idstr) +{ + while (sub && *sub) { + if (strcmp(idstr, (*sub)->name) == 0) { + return *sub; + } + sub++; + } + return NULL; +} + +static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque) +{ + trace_vmstate_subsection_load(vmsd->name); + + while (qemu_peek_byte(f, 0) == QEMU_VM_SUBSECTION) { + char idstr[256], *idstr_ret; + int ret; + uint8_t version_id, len, size; + const VMStateDescription *sub_vmsd; + + len = qemu_peek_byte(f, 1); + if (len < strlen(vmsd->name) + 1) { + /* subsection name has be be "section_name/a" */ + trace_vmstate_subsection_load_bad(vmsd->name, "(short)", ""); + return 0; + } + size = qemu_peek_buffer(f, (uint8_t **)&idstr_ret, len, 2); + if (size != len) { + trace_vmstate_subsection_load_bad(vmsd->name, "(peek fail)", ""); + return 0; + } + memcpy(idstr, idstr_ret, size); + idstr[size] = 0; + + if (strncmp(vmsd->name, idstr, strlen(vmsd->name)) != 0) { + trace_vmstate_subsection_load_bad(vmsd->name, idstr, "(prefix)"); + /* it doesn't have a valid subsection name */ + return 0; + } + sub_vmsd = vmstate_get_subsection(vmsd->subsections, idstr); + if (sub_vmsd == NULL) { + trace_vmstate_subsection_load_bad(vmsd->name, idstr, "(lookup)"); + return -ENOENT; + } + qemu_file_skip(f, 1); /* subsection */ + qemu_file_skip(f, 1); /* len */ + qemu_file_skip(f, len); /* idstr */ + version_id = qemu_get_be32(f); + + ret = vmstate_load_state(f, sub_vmsd, opaque, version_id); + if (ret) { + trace_vmstate_subsection_load_bad(vmsd->name, idstr, "(child)"); + return ret; + } + } + + trace_vmstate_subsection_load_good(vmsd->name); + return 0; +} + +static int vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd, + void *opaque, JSONWriter *vmdesc) +{ + const VMStateDescription **sub = vmsd->subsections; + bool vmdesc_has_subsections = false; + int ret = 0; + + trace_vmstate_subsection_save_top(vmsd->name); + while (sub && *sub) { + if (vmstate_save_needed(*sub, opaque)) { + const VMStateDescription *vmsdsub = *sub; + uint8_t len; + + trace_vmstate_subsection_save_loop(vmsd->name, vmsdsub->name); + if (vmdesc) { + /* Only create subsection array when we have any */ + if (!vmdesc_has_subsections) { + json_writer_start_array(vmdesc, "subsections"); + vmdesc_has_subsections = true; + } + + json_writer_start_object(vmdesc, NULL); + } + + qemu_put_byte(f, QEMU_VM_SUBSECTION); + len = strlen(vmsdsub->name); + qemu_put_byte(f, len); + qemu_put_buffer(f, (uint8_t *)vmsdsub->name, len); + qemu_put_be32(f, vmsdsub->version_id); + ret = vmstate_save_state(f, vmsdsub, opaque, vmdesc); + if (ret) { + return ret; + } + + if (vmdesc) { + json_writer_end_object(vmdesc); + } + } + sub++; + } + + if (vmdesc_has_subsections) { + json_writer_end_array(vmdesc); + } + + return ret; +} diff --git a/migration/xbzrle.c b/migration/xbzrle.c new file mode 100644 index 000000000..1ba482ded --- /dev/null +++ b/migration/xbzrle.c @@ -0,0 +1,176 @@ +/* + * Xor Based Zero Run Length Encoding + * + * Copyright 2013 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Orit Wasserman <owasserm@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ +#include "qemu/osdep.h" +#include "qemu/cutils.h" +#include "xbzrle.h" + +/* + page = zrun nzrun + | zrun nzrun page + + zrun = length + + nzrun = length byte... + + length = uleb128 encoded integer + */ +int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen) +{ + uint32_t zrun_len = 0, nzrun_len = 0; + int d = 0, i = 0; + long res; + uint8_t *nzrun_start = NULL; + + g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) % + sizeof(long))); + + while (i < slen) { + /* overflow */ + if (d + 2 > dlen) { + return -1; + } + + /* not aligned to sizeof(long) */ + res = (slen - i) % sizeof(long); + while (res && old_buf[i] == new_buf[i]) { + zrun_len++; + i++; + res--; + } + + /* word at a time for speed */ + if (!res) { + while (i < slen && + (*(long *)(old_buf + i)) == (*(long *)(new_buf + i))) { + i += sizeof(long); + zrun_len += sizeof(long); + } + + /* go over the rest */ + while (i < slen && old_buf[i] == new_buf[i]) { + zrun_len++; + i++; + } + } + + /* buffer unchanged */ + if (zrun_len == slen) { + return 0; + } + + /* skip last zero run */ + if (i == slen) { + return d; + } + + d += uleb128_encode_small(dst + d, zrun_len); + + zrun_len = 0; + nzrun_start = new_buf + i; + + /* overflow */ + if (d + 2 > dlen) { + return -1; + } + /* not aligned to sizeof(long) */ + res = (slen - i) % sizeof(long); + while (res && old_buf[i] != new_buf[i]) { + i++; + nzrun_len++; + res--; + } + + /* word at a time for speed, use of 32-bit long okay */ + if (!res) { + /* truncation to 32-bit long okay */ + unsigned long mask = (unsigned long)0x0101010101010101ULL; + while (i < slen) { + unsigned long xor; + xor = *(unsigned long *)(old_buf + i) + ^ *(unsigned long *)(new_buf + i); + if ((xor - mask) & ~xor & (mask << 7)) { + /* found the end of an nzrun within the current long */ + while (old_buf[i] != new_buf[i]) { + nzrun_len++; + i++; + } + break; + } else { + i += sizeof(long); + nzrun_len += sizeof(long); + } + } + } + + d += uleb128_encode_small(dst + d, nzrun_len); + /* overflow */ + if (d + nzrun_len > dlen) { + return -1; + } + memcpy(dst + d, nzrun_start, nzrun_len); + d += nzrun_len; + nzrun_len = 0; + } + + return d; +} + +int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen) +{ + int i = 0, d = 0; + int ret; + uint32_t count = 0; + + while (i < slen) { + + /* zrun */ + if ((slen - i) < 2) { + return -1; + } + + ret = uleb128_decode_small(src + i, &count); + if (ret < 0 || (i && !count)) { + return -1; + } + i += ret; + d += count; + + /* overflow */ + if (d > dlen) { + return -1; + } + + /* nzrun */ + if ((slen - i) < 2) { + return -1; + } + + ret = uleb128_decode_small(src + i, &count); + if (ret < 0 || !count) { + return -1; + } + i += ret; + + /* overflow */ + if (d + count > dlen || i + count > slen) { + return -1; + } + + memcpy(dst + d, src + i, count); + d += count; + i += count; + } + + return d; +} diff --git a/migration/xbzrle.h b/migration/xbzrle.h new file mode 100644 index 000000000..a0db507b9 --- /dev/null +++ b/migration/xbzrle.h @@ -0,0 +1,21 @@ +/* + * QEMU live migration + * + * Copyright IBM, Corp. 2008 + * + * Authors: + * Anthony Liguori <aliguori@us.ibm.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#ifndef QEMU_MIGRATION_XBZRLE_H +#define QEMU_MIGRATION_XBZRLE_H + +int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen, + uint8_t *dst, int dlen); + +int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen); +#endif diff --git a/migration/yank_functions.c b/migration/yank_functions.c new file mode 100644 index 000000000..8c08aef14 --- /dev/null +++ b/migration/yank_functions.c @@ -0,0 +1,62 @@ +/* + * migration yank functions + * + * Copyright (c) Lukas Straub <lukasstraub2@web.de> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qapi/error.h" +#include "io/channel.h" +#include "yank_functions.h" +#include "qemu/yank.h" +#include "io/channel-socket.h" +#include "io/channel-tls.h" +#include "qemu-file.h" + +void migration_yank_iochannel(void *opaque) +{ + QIOChannel *ioc = QIO_CHANNEL(opaque); + + qio_channel_shutdown(ioc, QIO_CHANNEL_SHUTDOWN_BOTH, NULL); +} + +/* Return whether yank is supported on this ioc */ +static bool migration_ioc_yank_supported(QIOChannel *ioc) +{ + return object_dynamic_cast(OBJECT(ioc), TYPE_QIO_CHANNEL_SOCKET) || + object_dynamic_cast(OBJECT(ioc), TYPE_QIO_CHANNEL_TLS); +} + +void migration_ioc_register_yank(QIOChannel *ioc) +{ + if (migration_ioc_yank_supported(ioc)) { + yank_register_function(MIGRATION_YANK_INSTANCE, + migration_yank_iochannel, + QIO_CHANNEL(ioc)); + } +} + +void migration_ioc_unregister_yank(QIOChannel *ioc) +{ + if (migration_ioc_yank_supported(ioc)) { + yank_unregister_function(MIGRATION_YANK_INSTANCE, + migration_yank_iochannel, + QIO_CHANNEL(ioc)); + } +} + +void migration_ioc_unregister_yank_from_file(QEMUFile *file) +{ + QIOChannel *ioc = qemu_file_get_ioc(file); + + if (ioc) { + /* + * For migration qemufiles, we'll always reach here. Though we'll skip + * calls from e.g. savevm/loadvm as they don't use yank. + */ + migration_ioc_unregister_yank(ioc); + } +} diff --git a/migration/yank_functions.h b/migration/yank_functions.h new file mode 100644 index 000000000..a7577955e --- /dev/null +++ b/migration/yank_functions.h @@ -0,0 +1,20 @@ +/* + * migration yank functions + * + * Copyright (c) Lukas Straub <lukasstraub2@web.de> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +/** + * migration_yank_iochannel: yank function for iochannel + * + * This yank function will call qio_channel_shutdown on the provided QIOChannel. + * + * @opaque: QIOChannel to shutdown + */ +void migration_yank_iochannel(void *opaque); +void migration_ioc_register_yank(QIOChannel *ioc); +void migration_ioc_unregister_yank(QIOChannel *ioc); +void migration_ioc_unregister_yank_from_file(QEMUFile *file); |