diff --git a/src/arch/riscv/prefix/libprefix.S b/src/arch/riscv/prefix/libprefix.S index 7f4e2bc98..c43a2b1b2 100644 --- a/src/arch/riscv/prefix/libprefix.S +++ b/src/arch/riscv/prefix/libprefix.S @@ -41,6 +41,13 @@ prefix_virt: .dword _prefix .size prefix_virt, . - prefix_virt + /* Current virtual address offset */ + .section ".data.virt_offset", "aw", @progbits + .globl virt_offset +virt_offset: + .space ( __riscv_xlen / 8 ) + .size virt_offset, . - virt_offset + /***************************************************************************** * * Print message to debug console diff --git a/src/arch/x86/image/elfboot.c b/src/arch/x86/image/elfboot.c index 7f89e8b65..d0f91d1c0 100644 --- a/src/arch/x86/image/elfboot.c +++ b/src/arch/x86/image/elfboot.c @@ -26,11 +26,11 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include #include #include +#include #include #include #include #include -#include /** * @file diff --git a/src/arch/x86/include/bits/uaccess.h b/src/arch/x86/include/bits/uaccess.h deleted file mode 100644 index e9e7e5af5..000000000 --- a/src/arch/x86/include/bits/uaccess.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _BITS_UACCESS_H -#define _BITS_UACCESS_H - -/** @file - * - * x86-specific user access API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include - -#endif /* _BITS_UACCESS_H */ diff --git a/src/arch/x86/include/librm.h b/src/arch/x86/include/librm.h index 4fce7e8c8..cbe7d7134 100644 --- a/src/arch/x86/include/librm.h +++ b/src/arch/x86/include/librm.h @@ -64,12 +64,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #else /* ASSEMBLY */ -#ifdef UACCESS_LIBRM -#define UACCESS_PREFIX_librm -#else -#define UACCESS_PREFIX_librm __librm_ -#endif - /** * Call C function from real-mode code * @@ -79,53 +73,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); "pushl $( " _S2 ( VIRTUAL ( function ) ) " )\n\t" \ "call virt_call\n\t" -/* Variables in librm.S */ -extern const unsigned long virt_offset; - -/** - * Convert physical address to user pointer - * - * @v phys Physical address - * @ret virt Virtual address - */ -static inline __always_inline void * -UACCESS_INLINE ( librm, phys_to_virt ) ( unsigned long phys ) { - - /* In a 64-bit build, any valid physical address is directly - * usable as a virtual address, since the low 4GB is - * identity-mapped. - */ - if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) - return ( ( void * ) phys ); - - /* In a 32-bit build, subtract virt_offset */ - return ( ( void * ) ( phys - virt_offset ) ); -} - -/** - * Convert virtual address to physical address - * - * @v virt Virtual address - * @ret phys Physical address - */ -static inline __always_inline physaddr_t -UACCESS_INLINE ( librm, virt_to_phys ) ( volatile const void *virt ) { - physaddr_t addr = ( ( physaddr_t ) virt ); - - /* In a 64-bit build, any virtual address in the low 4GB is - * directly usable as a physical address, since the low 4GB is - * identity-mapped. - */ - if ( ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) && - ( addr <= 0xffffffffUL ) ) - return addr; - - /* In a 32-bit build or in a 64-bit build with a virtual - * address above 4GB: add virt_offset - */ - return ( addr + virt_offset ); -} - /****************************************************************************** * * Access to variables in .data16 and .text16 diff --git a/src/arch/x86/include/realmode.h b/src/arch/x86/include/realmode.h index 5cf644a23..7baec56ca 100644 --- a/src/arch/x86/include/realmode.h +++ b/src/arch/x86/include/realmode.h @@ -4,6 +4,7 @@ #include #include #include +#include #include /* diff --git a/src/arch/x86/interface/pcbios/bios_mp.c b/src/arch/x86/interface/pcbios/bios_mp.c index c903a81fc..4525a60cf 100644 --- a/src/arch/x86/interface/pcbios/bios_mp.c +++ b/src/arch/x86/interface/pcbios/bios_mp.c @@ -30,6 +30,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); */ #include +#include #include #include #include diff --git a/src/arch/x86/transitions/librm_mgmt.c b/src/arch/x86/transitions/librm_mgmt.c index 14e00eda8..0328cb2a5 100644 --- a/src/arch/x86/transitions/librm_mgmt.c +++ b/src/arch/x86/transitions/librm_mgmt.c @@ -429,8 +429,6 @@ void setup_sipi ( unsigned int vector, uint32_t handler, copy_to_real ( ( vector << 8 ), 0, sipi, ( ( size_t ) sipi_len ) ); } -PROVIDE_UACCESS_INLINE ( librm, phys_to_virt ); -PROVIDE_UACCESS_INLINE ( librm, virt_to_phys ); PROVIDE_IOMAP_INLINE ( pages, io_to_bus ); PROVIDE_IOMAP ( pages, ioremap, ioremap_pages ); PROVIDE_IOMAP ( pages, iounmap, iounmap_pages ); diff --git a/src/config/defaults/pcbios.h b/src/config/defaults/pcbios.h index fa12a1005..13532e429 100644 --- a/src/config/defaults/pcbios.h +++ b/src/config/defaults/pcbios.h @@ -9,7 +9,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); -#define UACCESS_LIBRM +#define UACCESS_OFFSET #define IOAPI_X86 #define PCIAPI_PCBIOS #define DMAAPI_FLAT diff --git a/src/config/defaults/sbi.h b/src/config/defaults/sbi.h index 27fbe0e80..268bf0afe 100644 --- a/src/config/defaults/sbi.h +++ b/src/config/defaults/sbi.h @@ -12,7 +12,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #define IOAPI_RISCV #define IOMAP_VIRT #define DMAAPI_FLAT -#define UACCESS_FLAT +#define UACCESS_OFFSET #define TIMER_ZICNTR #define ENTROPY_ZKR diff --git a/src/core/virt_offset.c b/src/core/virt_offset.c new file mode 100644 index 000000000..23862d934 --- /dev/null +++ b/src/core/virt_offset.c @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2025 Michael Brown . + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include + +/** + * @file + * + * Virtual offset memory model + * + */ + +PROVIDE_UACCESS_INLINE ( offset, phys_to_virt ); +PROVIDE_UACCESS_INLINE ( offset, virt_to_phys ); diff --git a/src/include/ipxe/uaccess.h b/src/include/ipxe/uaccess.h index 8d93cbf4f..d97db95be 100644 --- a/src/include/ipxe/uaccess.h +++ b/src/include/ipxe/uaccess.h @@ -60,6 +60,7 @@ UACCESS_INLINE ( flat, virt_to_phys ) ( volatile const void *virt ) { } /* Include all architecture-independent user access API headers */ +#include #include /* Include all architecture-dependent user access API headers */ diff --git a/src/include/ipxe/virt_offset.h b/src/include/ipxe/virt_offset.h new file mode 100644 index 000000000..f8f902753 --- /dev/null +++ b/src/include/ipxe/virt_offset.h @@ -0,0 +1,140 @@ +#ifndef _IPXE_VIRT_OFFSET_H +#define _IPXE_VIRT_OFFSET_H + +/** + * @file + * + * Virtual offset memory model + * + * No currently supported machine provides a full 64 bits of physical + * address space. When we have ownership of the page tables (or + * segmentation mechanism), we can therefore use the following model: + * + * - For 32-bit builds: set up a circular map so that all 32-bit + * virtual addresses are at a fixed offset from the 32-bit + * physical addresses. + * + * - For 64-bit builds: identity-map the required portion of the + * physical address space, then map iPXE itself using virtual + * addresses in the negative (kernel) address space. + * + * In both cases, we can define "virt_offset" as "the value to be + * added to an address within iPXE's own image in order to obtain its + * physical address". With this definition: + * + * - For 32-bit builds: conversion between physical and virtual + * addresses is a straightforward addition or subtraction of + * virt_offset, since the whole 32-bit address space is circular. + * + * - For 64-bit builds: conversion from any valid physical address + * is a no-op (since all physical addresses are identity-mapped), + * and conversion from a virtual address to a physical address + * requires an addition of virt_offset if and only if the virtual + * address lies in the negative portion of the address space + * (i.e. has the MSB set). + * + * For x86_64-pcbios, we identity-map the low 4GB of address space + * since the only accesses required above 4GB are for MMIO (typically + * PCI devices with large memory BARs). + * + * For riscv64-sbi, we identity-map as much of the physical address + * space as can be mapped by the paging model (Sv39, Sv48, or Sv57). + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#ifdef UACCESS_OFFSET +#define UACCESS_PREFIX_offset +#else +#define UACCESS_PREFIX_offset __offset_ +#endif + +/** Virtual address offset + * + * This is defined to be the value to be added to an address within + * iPXE's own image in order to obtain its physical address, as + * described above. + * + * Note that if iPXE's image is not yet writable (i.e. during early + * startup, prior to physical relocation), then this value may not yet + * be valid. Under these circumstances, callers must use + * offset_phys_to_virt() and offset_virt_to_phys() instead (and so + * provide the virtual address offset as a function parameter). + */ +extern const unsigned long virt_offset; + +/** + * Convert physical address to virtual address + * + * @v phys Physical address + * @v offset Virtual address offset + * @ret virt Virtual address + */ +static inline __always_inline void * +offset_phys_to_virt ( unsigned long phys, unsigned long offset ) { + + /* In a 64-bit build, any valid physical address is directly + * usable as a virtual address, since physical addresses are + * identity-mapped. + */ + if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) + return ( ( void * ) phys ); + + /* In a 32-bit build: subtract virt_offset */ + return ( ( void * ) ( phys - offset ) ); +} + +/** + * Convert virtual address to physical address + * + * @v virt Virtual address + * @v offset Virtual address offset + * @ret phys Physical address + */ +static inline __always_inline physaddr_t +offset_virt_to_phys ( volatile const void *virt, unsigned long offset ) { + physaddr_t addr = ( ( physaddr_t ) virt ); + + /* In a 64-bit build, any valid virtual address with the MSB + * clear is directly usable as a physical address, since it + * must lie within the identity-mapped portion. + * + * This test will typically reduce to a single "branch if less + * than zero" instruction. + */ + if ( ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) && + ( ! ( addr & ( 1ULL << ( 8 * sizeof ( physaddr_t ) - 1 ) ) ) ) ) { + return addr; + } + + /* In a 32-bit build or in a 64-bit build with a virtual + * address with the MSB set: add virt_offset + */ + return ( addr + offset ); +} + +/** + * Convert physical address to virtual address + * + * @v phys Physical address + * @ret virt Virtual address + */ +static inline __always_inline void * +UACCESS_INLINE ( offset, phys_to_virt ) ( unsigned long phys ) { + + return offset_phys_to_virt ( phys, virt_offset ); +} + +/** + * Convert virtual address to physical address + * + * @v virt Virtual address + * @ret phys Physical address + */ +static inline __always_inline physaddr_t +UACCESS_INLINE ( offset, virt_to_phys ) ( volatile const void *virt ) { + + return offset_virt_to_phys ( virt, virt_offset ); +} + +#endif /* _IPXE_VIRT_OFFSET_H */