Paravirtualization

┌──────────────────────────────────────────────────────────────────────┐
│                           Applications                                │
├──────────────────────────────────────────────────────────────────────┤
│    ┌─────────────┐    ┌─────────────┐    ┌─────────────┐             │
│    │   Dom0      │    │   DomU-1    │    │   DomU-2    │             │
│    │  (Linux)    │    │  (Linux)    │    │  (NetBSD)   │             │
│    │             │    │             │    │             │             │
│    │ ┌─────────┐ │    │ ┌─────────┐ │    │ ┌─────────┐ │             │
│    │ │ Backend │ │    │ │Frontend │ │    │ │Frontend │ │             │
│    │ │ Drivers │ │    │ │ Drivers │ │    │ │ Drivers │ │             │
│    │ └────┬────┘ │    │ └────┬────┘ │    │ └────┬────┘ │             │
│    │      │      │    │      │      │    │      │      │             │
│    │   Native    │    │   Paravirt  │    │   Paravirt  │             │
│    │   Drivers   │    │   Kernel    │    │   Kernel    │             │
│    └──────┼──────┘    └──────┼──────┘    └──────┼──────┘             │
│           │                  │                  │                     │
│    ┌──────┴──────────────────┴──────────────────┴──────┐             │
│    │              Event Channels / Grant Tables         │             │
│    └────────────────────────┬───────────────────────────┘             │
├─────────────────────────────┴────────────────────────────────────────┤
│                         Xen Hypervisor                               │
│    ┌───────────┐  ┌───────────┐  ┌───────────┐  ┌───────────┐       │
│    │  vCPU     │  │  Memory   │  │  Event    │  │  Grant    │       │
│    │ Scheduler │  │ Management│  │ Channels  │  │  Tables   │       │
│    └───────────┘  └───────────┘  └───────────┘  └───────────┘       │
├──────────────────────────────────────────────────────────────────────┤
│                           Hardware                                   │
│    ┌───────────┐  ┌───────────┐  ┌───────────┐  ┌───────────┐       │
│    │    CPU    │  │   Memory  │  │   NIC     │  │   Disk    │       │
│    └───────────┘  └───────────┘  └───────────┘  └───────────┘       │
└──────────────────────────────────────────────────────────────────────┘

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

/* Xen Memory Virtualization Details */
 
/*
 * Physical Memory Map:
 * 
 *   Guest's View (PFN - Pseudo-Physical):
 *   ┌────────────────────────┐
 *   │ PFN 0: Guest RAM       │
 *   │ PFN 1: Guest RAM       │
 *   │ ...                    │
 *   │ PFN N: Guest RAM       │
 *   └────────────────────────┘
 *   Contiguous, starting at 0
 * 
 *   Machine's View (MFN - Machine Frame):
 *   ┌────────────────────────┐
 *   │ MFN 0: Xen              │
 *   │ MFN 1: Dom0            │
 *   │ MFN 2: DomU-1          │  ← Guest PFN 0
 *   │ MFN 3: Dom0            │
 *   │ MFN 4: DomU-1          │  ← Guest PFN 1
 *   │ ...                    │
 *   └────────────────────────┘
 *   Scattered across physical memory
 */
 
/* Translation tables - bidirectional mapping */
extern unsigned long *phys_to_machine_mapping;  /* PFN → MFN */
/* MFN → PFN via shared page from Xen */
#define machine_to_phys_mapping ((unsigned long *)MACH2PHYS_VIRT_START)
 
/* Convert guest physical address to machine address */
static inline unsigned long phys_to_machine(unsigned long phys) {
    unsigned long pfn = phys >> PAGE_SHIFT;
    unsigned long offset = phys & ~PAGE_MASK;
    unsigned long mfn = pfn_to_mfn(pfn);
    return (mfn << PAGE_SHIFT) | offset;
}
 
/* Page table entry construction - must use MFN, not PFN */
static inline pte_t xen_make_pte(unsigned long val) {
    if (val & _PAGE_PRESENT) {
        unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT;
        unsigned long mfn = pfn_to_mfn(pfn);
        val = (val & ~PTE_PFN_MASK) | (mfn << PAGE_SHIFT);
    }
    return (pte_t) { .pte = val };
}
 
/*
 * Page Table Registration:
 * 
 * Before page tables can be used, they must be "pinned" with Xen.
 * This validates the entire table and marks pages as page-table pages.
 */
int xen_pin_page_table(unsigned long pfn, unsigned int level) {
    struct mmuext_op op;
    
    op.cmd = MMUEXT_PIN_L1_TABLE + level;  /* Level 1-4 */
    op.arg1.mfn = pfn_to_mfn(pfn);
    
    return HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF);
}
 
/* 
 * Validation enforces:
 * - All MFNs in PTEs must be owned by the domain (or granted)
 * - Page table pages cannot be mapped writable
 * - Hypervisor memory cannot be mapped
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

/* Xen PV Interrupt Handling */
 
/*
 * Callback registration - done during guest boot
 */
struct xen_callback_register {
    uint16_t type;
    uint16_t flags;
    xen_callback_t address;
};
 
#define CALLBACKTYPE_event              0  /* Event channel upcall */
#define CALLBACKTYPE_failsafe           1  /* Critical failure handler */
#define CALLBACKTYPE_syscall            2  /* System call entry */
#define CALLBACKTYPE_nmi                4  /* NMI delivery */
 
void __init xen_init_callbacks(void) {
    /* Register event channel callback */
    struct xen_callback_register cb = {
        .type = CALLBACKTYPE_event,
        .address = (xen_callback_t)xen_hypervisor_callback,
    };
    HYPERVISOR_callback_op(CALLBACKOP_register, &cb);
    
    /* Register exception (fault) callback */
    cb.type = CALLBACKTYPE_failsafe;
    cb.address = (xen_callback_t)xen_failsafe_callback;
    HYPERVISOR_callback_op(CALLBACKOP_register, &cb);
    
    /* Register syscall entry for PV guests */
    cb.type = CALLBACKTYPE_syscall;
    cb.address = (xen_callback_t)xen_syscall_target;
    HYPERVISOR_callback_op(CALLBACKOP_register, &cb);
}
 
/*
 * Exception delivery:
 * 
 * When guest causes page fault:
 * 1. CPU traps to Xen (Ring 1 → Ring 0)
 * 2. Xen saves guest state
 * 3. Xen calls registered exception callback
 * 4. Guest handler runs
 * 5. Guest returns to Xen via hypercall
 * 6. Xen resumes guest at fault point
 */
 
/* Trap info structure - replaces IDT entries */
struct trap_info {
    uint8_t  vector;    /* Exception vector (0-255) */
    uint8_t  flags;     /* Privilege level, type */
    uint16_t cs;        /* Code segment selector */
    unsigned long address;  /* Handler address */
};
 
void xen_set_trap_table(struct trap_info *table) {
    /* Register exception handlers with Xen */
    HYPERVISOR_set_trap_table(table);
}
 
/* The callback that Xen invokes for events */
__visible void xen_hypervisor_callback(void) {
    struct pt_regs *regs = get_irq_regs();
    struct shared_info *s = HYPERVISOR_shared_info;
    struct vcpu_info *v = this_cpu_ptr(xen_vcpu);
    
    v->evtchn_upcall_pending = 0;
    
    /* Process all pending event channels */
    xen_evtchn_do_upcall(regs);
    
    /* Return to interrupted context */
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

/* Xen Event Channel Implementation */
 
/*
 * Event Channel State (per-domain):
 * 
 * shared_info page (mapped into guest):
 *   - evtchn_pending[]: Bitmap of pending events
 *   - evtchn_mask[]: Bitmap of masked events
 * 
 * vcpu_info (per-vCPU):
 *   - evtchn_upcall_pending: Flag indicating pending events
 *   - evtchn_upcall_mask: Master interrupt disable
 *   - evtchn_pending_sel: Selector for pending words
 */
 
struct shared_info {
    struct vcpu_info vcpu_info[MAX_VIRT_CPUS];
    
    /* Bitfield of pending events per port */
    unsigned long evtchn_pending[sizeof(unsigned long) * 8];
    
    /* Bitfield of masked events per port */
    unsigned long evtchn_mask[sizeof(unsigned long) * 8];
    
    /* Wall clock time */
    uint32_t wc_sec;
    uint32_t wc_nsec;
    uint32_t wc_sec_hi;
    
    /* Architecture-specific shared info */
    struct arch_shared_info arch;
};
 
/* Binding a VIRQ (virtual IRQ) to an event channel */
int bind_virq_to_evtchn(unsigned int virq) {
    struct evtchn_bind_virq bind = {
        .virq = virq,
        .vcpu = smp_processor_id(),
    };
    
    if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind))
        return -1;
    
    return bind.port;  /* Allocated event channel port */
}
 
/* Sending notification on an interdomain channel */
void notify_via_evtchn(unsigned int port) {
    struct evtchn_send send = { .port = port };
    HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
}
 
/* 
 * Event Channel Port Binding (Frontend → Backend example):
 * 
 * 1. Backend (Dom0) allocates unbound channel:
 *    evtchn_alloc_unbound(DOMID_SELF, frontend_domid)
 *    → Returns port P_backend
 * 
 * 2. Backend writes port to XenStore:
 *    xenstore_write("device/vbd/0/event-channel", P_backend)
 * 
 * 3. Frontend (DomU) reads port and binds:
 *    evtchn_bind_interdomain(backend_domid, P_backend)
 *    → Returns port P_frontend
 * 
 * 4. Now both ends can notify each other:
 *    Backend: notify_via_evtchn(P_backend)
 *    Frontend receives event on P_frontend
 */
 
/* Creating interdomain channel (frontend side) */
int bind_interdomain(domid_t remote_dom, unsigned int remote_port) {
    struct evtchn_bind_interdomain bind = {
        .remote_dom = remote_dom,
        .remote_port = remote_port,
    };
    
    if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain, &bind))
        return -1;
    
    return bind.local_port;
}
 
/* IPI via event channel */
void xen_send_IPI(int cpu, int vector) {
    int evtchn = per_cpu(ipi_evtchn, cpu)[vector];
    notify_via_evtchn(evtchn);
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

/* Xen Grant Table Mechanism */
 
/*
 * Grant Entry Structure:
 * 
 * Each domain has a grant table - an array of grant entries.
 * Grant references are indices into this table.
 */
struct grant_entry_v1 {
    uint16_t flags;        /* GTF_* flags (permit_access, etc.) */
    domid_t  domid;        /* Domain being granted access */
    uint32_t frame;        /* MFN of the granted page */
};
 
#define GTF_invalid         (0)
#define GTF_permit_access   (1)
#define GTF_accept_transfer (2)
#define GTF_reading         (1 << 3)
#define GTF_writing         (1 << 4)
 
/* Granting side: Share a page with another domain */
grant_ref_t gnttab_grant_access(domid_t domid, unsigned long pfn, 
                                 bool readonly) {
    grant_ref_t ref;
    struct grant_entry_v1 *entry;
    
    /* Allocate an unused grant reference */
    ref = get_free_grant_ref();
    entry = &grant_table[ref];
    
    /* Set up the grant entry */
    entry->domid = domid;
    entry->frame = pfn_to_mfn(pfn);
    wmb();  /* Ensure domid and frame visible before flags */
    
    entry->flags = GTF_permit_access | (readonly ? GTF_reading : 
                                        (GTF_reading | GTF_writing));
    
    return ref;
}
 
/* Receiving side: Map a granted page */
void *gnttab_map_grant_ref(domid_t remote_dom, grant_ref_t ref) {
    struct gnttab_map_grant_ref map = {
        .host_addr = get_free_vm_area(PAGE_SIZE),
        .flags = GNTMAP_host_map,
        .ref = ref,
        .dom = remote_dom,
    };
    
    if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, &map, 1))
        return NULL;
    
    if (map.status != GNTST_okay)
        return NULL;
    
    /* Page is now mapped at map.host_addr */
    return (void *)map.host_addr;
}
 
/* Receiving side: Unmap when done */
void gnttab_unmap_grant_ref(void *addr, grant_handle_t handle) {
    struct gnttab_unmap_grant_ref unmap = {
        .host_addr = (unsigned long)addr,
        .handle = handle,
    };
    
    HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmap, 1);
}
 
/* Granting side: Revoke access */
void gnttab_end_access(grant_ref_t ref) {
    struct grant_entry_v1 *entry = &grant_table[ref];
    
    /* Clear flags to revoke (must wait for unmap) */
    entry->flags = GTF_invalid;
    mb();
    
    /* Return ref to free pool */
    put_free_grant_ref(ref);
}
 
/*
 * Usage in Split Driver (Block Device Example):
 * 
 * Frontend (DomU):
 *   1. Allocate page for data
 *   2. Grant access: ref = gnttab_grant_access(dom0, pfn, false)
 *   3. Put ref in block request: request.gref = ref
 *   4. Send request via ring + event channel
 * 
 * Backend (Dom0):
 *   1. Receive request from ring
 *   2. Map granted page: addr = gnttab_map_grant_ref(domu, ref)
 *   3. Perform I/O: read/write(disk_fd, addr, len)
 *   4. Unmap page: gnttab_unmap_grant_ref(addr, handle)
 *   5. Send completion via ring + event channel
 * 
 * Result: Zero-copy - data never duplicated between domains
 */

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

/* XenStore Operations */
 
/*
 * XenStore Namespace Example:
 * 
 * /local/domain/0/                      (Dom0's area)
 * /local/domain/0/device-model          (QEMU info)
 * 
 * /local/domain/1/                      (DomU 1's area)
 * /local/domain/1/device/vbd/768/       (Block device)
 * /local/domain/1/device/vbd/768/backend (Path to backend)
 * /local/domain/1/device/vbd/768/state  (Device state)
 * 
 * /local/domain/0/backend/vbd/1/768/    (Backend for DomU 1)
 * /local/domain/0/backend/vbd/1/768/frontend (Path to frontend)
 * /local/domain/0/backend/vbd/1/768/ring-ref (Grant ref for ring)
 * /local/domain/0/backend/vbd/1/768/event-channel (Event channel port)
 */
 
/* Reading a XenStore key */
char *xenstore_read(const char *path) {
    char *value;
    unsigned int len;
    
    value = xenbus_read(XBT_NIL, path, &len);
    return value;  /* Caller must free */
}
 
/* Writing a XenStore key */
int xenstore_write(const char *path, const char *value) {
    return xenbus_write(XBT_NIL, path, value);
}
 
/* Watching for changes */
int xenstore_watch(const char *path, struct xenbus_watch *watch,
                   void (*callback)(struct xenbus_watch *, const char *, 
                                   const char *)) {
    watch->node = path;
    watch->callback = callback;
    return register_xenbus_watch(watch);
}
 
/*
 * Device Connection Protocol via XenStore:
 * 
 * 1. Toolstack creates device entries:
 *    /local/domain/1/device/vbd/768/backend = /local/domain/0/backend/vbd/1/768
 *    /local/domain/0/backend/vbd/1/768/frontend = /local/domain/1/device/vbd/768
 * 
 * 2. Backend driver watches its area, sees new device
 * 
 * 3. Backend initializes:
 *    - Allocates ring buffer page
 *    - Grants page to frontend domain
 *    - Allocates event channel
 *    - Writes ring-ref, event-channel to XenStore
 *    - Sets state = XenbusStateInitWait
 * 
 * 4. Frontend driver watches its device area
 * 
 * 5. Frontend sees backend ready:
 *    - Reads ring-ref, event-channel from XenStore
 *    - Maps ring page using grant tables
 *    - Binds event channel
 *    - Sets state = XenbusStateConnected
 * 
 * 6. Backend sees frontend connected:
 *    - Sets state = XenbusStateConnected
 *    - Device is now operational
 */
 
/* State machine for XenBus device connection */
enum xenbus_state {
    XenbusStateUnknown      = 0,
    XenbusStateInitialising = 1,
    XenbusStateInitWait     = 2,
    XenbusStateInitialised  = 3,
    XenbusStateConnected    = 4,
    XenbusStateClosing      = 5,
    XenbusStateClosed       = 6,
    XenbusStateReconfiguring = 7,
    XenbusStateReconfigured = 8,
};
 
/* Example: Backend driver initialization */
int backend_probe(struct xenbus_device *dev) {
    struct backend_info *bi;
    int err;
    
    /* Allocate private data */
    bi = kzalloc(sizeof(*bi), GFP_KERNEL);
    
    /* Read frontend info */
    bi->frontend_id = dev->otherend_id;
    
    /* Set up ring buffer page */
    bi->ring_page = alloc_page(GFP_KERNEL);
    bi->ring_ref = gnttab_grant_access(bi->frontend_id,
                                       page_to_pfn(bi->ring_page), false);
    
    /* Allocate event channel */
    err = evtchn_alloc_unbound(DOMID_SELF, bi->frontend_id, 
                               &bi->evtchn);
    
    /* Publish to XenStore */
    xenstore_write_int(dev->nodename, "ring-ref", bi->ring_ref);
    xenstore_write_int(dev->nodename, "event-channel", bi->evtchn);
    
    /* Signal ready for frontend */
    xenbus_switch_state(dev, XenbusStateInitWait);
    
    return 0;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

# Xen Configuration Examples
 
# === Pure PV Guest (Legacy) ===
name = "pv-guest"
type = "pv"                     # Paravirtualized mode
kernel = "/boot/vmlinuz"        # PV kernel required
ramdisk = "/boot/initrd.img"
memory = 2048
vcpus = 2
disk = ['file:/var/lib/xen/images/disk.img,xvda,w']
vif = ['bridge=xenbr0']
 
# === HVM Guest (for Windows) ===
name = "hvm-guest"
type = "hvm"                    # Hardware virtualized
builder = "hvm"
memory = 4096
vcpus = 4
disk = ['file:/var/lib/xen/images/win.img,hda,w']
vif = ['model=e1000,bridge=xenbr0']  # Emulated NIC
boot = "d"
cdrom = "/path/to/windows.iso"
sdl = 0
vnc = 1
stdvga = 1
 
# === PVH Guest (Recommended for Linux) ===
name = "pvh-guest"
type = "pvh"                    # PV-in-HVM mode
kernel = "/boot/vmlinuz"        # Can use same kernel as PV
ramdisk = "/boot/initrd.img"
memory = 4096
vcpus = 4
disk = ['file:/var/lib/xen/images/disk.img,xvda,w']
vif = ['bridge=xenbr0']         # PV network driver
# No device emulation needed - uses PV drivers
# No BIOS emulation - direct boot

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

#!/bin/bash
# Common Xen Management Operations
 
# === Domain Lifecycle ===
 
# Create a new domain from config
xl create /etc/xen/guest.cfg
 
# List running domains
xl list
# Name       ID   Mem VCPUs    State   Time(s)
# Domain-0    0  4096     4   r-----    1234.5
# guest1      1  2048     2   -b----     567.8
 
# Console access
xl console guest1
 
# Pause/unpause execution
xl pause guest1
xl unpause guest1
 
# Shutdown gracefully
xl shutdown guest1
 
# Force destroy
xl destroy guest1
 
# === Live Migration ===
 
# Migrate to another host
xl migrate guest1 target-host.example.com
 
# With secure migration (TLS)
xl migrate -s guest1 target-host.example.com
 
# === Resource Management ===
 
# Adjust memory (balloon)
xl mem-set guest1 4096m
 
# Pin vCPUs to physical CPUs
xl vcpu-pin guest1 0 2      # vCPU 0 → pCPU 2
xl vcpu-pin guest1 1 3      # vCPU 1 → pCPU 3
 
# Adjust vCPU count online
xl vcpu-set guest1 4
 
# === Networking ===
 
# Add network interface
xl network-attach guest1 bridge=xenbr0
 
# Remove network interface
xl network-detach guest1 0
 
# === Storage ===
 
# Attach block device
xl block-attach guest1 phy:/dev/sdb1 xvdb w
 
# Detach block device
xl block-detach guest1 xvdb
 
# === Diagnostics ===
 
# Domain info
xl info
xl dominfo guest1
 
# VM configuration dump
xl config guest1
 
# Xen hypervisor message buffer
xl dmesg
 
# Performance stats
xl top