diff --git a/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs b/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs
index a807eed51..5610b7749 100644
--- a/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs
+++ b/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs
@@ -157,7 +157,7 @@ namespace ARMeilleure.Instructions
context.Copy(temp, value);
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -198,7 +198,7 @@ namespace ARMeilleure.Instructions
SetInt(context, rt, value);
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -265,7 +265,7 @@ namespace ARMeilleure.Instructions
context.Copy(GetVec(rt), value);
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -312,7 +312,7 @@ namespace ARMeilleure.Instructions
break;
}
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -385,7 +385,7 @@ namespace ARMeilleure.Instructions
break;
}
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -403,6 +403,21 @@ namespace ARMeilleure.Instructions
{
return EmitHostMappedPointer(context, address);
}
+ else if (context.Memory.Type == MemoryManagerType.HostTracked)
+ {
+ Operand ptBase = !context.HasPtc
+ ? Const(context.Memory.PageTablePointer.ToInt64())
+ : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol);
+
+ Operand ptOffset = context.ShiftRightUI(address, Const(PageBits));
+
+ if (ptOffset.Type == OperandType.I32)
+ {
+ ptOffset = context.ZeroExtend32(OperandType.I64, ptOffset);
+ }
+
+ return context.Add(address, context.Load(OperandType.I64, context.Add(ptBase, context.ShiftLeft(ptOffset, Const(3)))));
+ }
int ptLevelBits = context.Memory.AddressSpaceBits - PageBits;
int ptLevelSize = 1 << ptLevelBits;
diff --git a/src/ARMeilleure/Memory/MemoryManagerType.cs b/src/ARMeilleure/Memory/MemoryManagerType.cs
index 1e656ba27..757322b4b 100644
--- a/src/ARMeilleure/Memory/MemoryManagerType.cs
+++ b/src/ARMeilleure/Memory/MemoryManagerType.cs
@@ -18,6 +18,12 @@ namespace ARMeilleure.Memory
/// </summary>
SoftwarePageTable,
+ /// <summary>
+ /// High level implementation using a software flat page table for address translation,
+ /// no support for handling invalid or non-contiguous memory access.
+ /// </summary>
+ HostTracked,
+
/// <summary>
/// High level implementation with mappings managed by the host OS, effectively using hardware
/// page tables. No address translation is performed in software and the memory is just accessed directly.
@@ -37,5 +43,10 @@ namespace ARMeilleure.Memory
{
return type == MemoryManagerType.HostMapped || type == MemoryManagerType.HostMappedUnsafe;
}
+
+ public static bool IsHostMappedOrTracked(this MemoryManagerType type)
+ {
+ return type == MemoryManagerType.HostTracked || type == MemoryManagerType.HostMapped || type == MemoryManagerType.HostMappedUnsafe;
+ }
}
}
diff --git a/src/ARMeilleure/Signal/NativeSignalHandler.cs b/src/ARMeilleure/Signal/NativeSignalHandler.cs
index 31ec16cb1..3f0e9e4bf 100644
--- a/src/ARMeilleure/Signal/NativeSignalHandler.cs
+++ b/src/ARMeilleure/Signal/NativeSignalHandler.cs
@@ -1,4 +1,4 @@
-using ARMeilleure.IntermediateRepresentation;
+using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.Memory;
using ARMeilleure.Translation;
using ARMeilleure.Translation.Cache;
diff --git a/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs b/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs
index 3957a7559..3c2a60a1a 100644
--- a/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs
+++ b/src/ARMeilleure/Translation/Cache/JitUnwindWindows.cs
@@ -114,7 +114,7 @@ namespace ARMeilleure.Translation.Cache
{
int stackOffset = entry.StackOffsetOrAllocSize;
- Debug.Assert(stackOffset % 16 == 0);
+ // Debug.Assert(stackOffset % 16 == 0);
if (stackOffset <= 0xFFFF0)
{
@@ -135,7 +135,7 @@ namespace ARMeilleure.Translation.Cache
{
int allocSize = entry.StackOffsetOrAllocSize;
- Debug.Assert(allocSize % 8 == 0);
+ // Debug.Assert(allocSize % 8 == 0);
if (allocSize <= 128)
{
diff --git a/src/ARMeilleure/Translation/Translator.cs b/src/ARMeilleure/Translation/Translator.cs
index 3ad76b0f7..184880ee4 100644
--- a/src/ARMeilleure/Translation/Translator.cs
+++ b/src/ARMeilleure/Translation/Translator.cs
@@ -80,7 +80,7 @@ namespace ARMeilleure.Translation
FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub;
- if (memory.Type.IsHostMapped())
+ if (memory.Type.IsHostMappedOrTracked())
{
NativeSignalHandler.InitializeSignalHandler(allocator.GetPageSize());
}
diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
index 4e3723d55..86936c592 100644
--- a/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
+++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
@@ -38,7 +38,7 @@ namespace Ryujinx.Cpu.AppleHv
private readonly HvIpaAllocator _ipaAllocator;
- public HvMemoryBlockAllocator(HvIpaAllocator ipaAllocator, int blockAlignment) : base(blockAlignment, MemoryAllocationFlags.None)
+ public HvMemoryBlockAllocator(HvIpaAllocator ipaAllocator, ulong blockAlignment) : base(blockAlignment, MemoryAllocationFlags.None)
{
_ipaAllocator = ipaAllocator;
}
diff --git a/src/Ryujinx.Cpu/Jit/AddressSpacePartition.cs b/src/Ryujinx.Cpu/Jit/AddressSpacePartition.cs
new file mode 100644
index 000000000..da3b2a3d2
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/AddressSpacePartition.cs
@@ -0,0 +1,612 @@
+using Ryujinx.Common;
+using Ryujinx.Common.Collections;
+using Ryujinx.Memory;
+using System;
+using System.Diagnostics;
+
+namespace Ryujinx.Cpu.Jit
+{
+ readonly struct PrivateRange
+ {
+ public readonly MemoryBlock Memory;
+ public readonly ulong Offset;
+ public readonly ulong Size;
+
+ public static PrivateRange Empty => new(null, 0, 0);
+
+ public PrivateRange(MemoryBlock memory, ulong offset, ulong size)
+ {
+ Memory = memory;
+ Offset = offset;
+ Size = size;
+ }
+ }
+
+ class AddressSpacePartition : IDisposable
+ {
+ private const ulong GuestPageSize = 0x1000;
+
+ private const int DefaultBlockAlignment = 1 << 20;
+
+ private enum MappingType : byte
+ {
+ None,
+ Private,
+ }
+
+ private class Mapping : IntrusiveRedBlackTreeNode<Mapping>, IComparable<Mapping>
+ {
+ public ulong Address { get; private set; }
+ public ulong Size { get; private set; }
+ public ulong EndAddress => Address + Size;
+ public MappingType Type { get; private set; }
+
+ public Mapping(ulong address, ulong size, MappingType type)
+ {
+ Address = address;
+ Size = size;
+ Type = type;
+ }
+
+ public Mapping Split(ulong splitAddress)
+ {
+ ulong leftSize = splitAddress - Address;
+ ulong rightSize = EndAddress - splitAddress;
+
+ Mapping left = new(Address, leftSize, Type);
+
+ Address = splitAddress;
+ Size = rightSize;
+
+ return left;
+ }
+
+ public void UpdateState(MappingType newType)
+ {
+ Type = newType;
+ }
+
+ public void Extend(ulong sizeDelta)
+ {
+ Size += sizeDelta;
+ }
+
+ public int CompareTo(Mapping other)
+ {
+ if (Address < other.Address)
+ {
+ return -1;
+ }
+ else if (Address <= other.EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ }
+
+ private class PrivateMapping : IntrusiveRedBlackTreeNode<PrivateMapping>, IComparable<PrivateMapping>
+ {
+ public ulong Address { get; private set; }
+ public ulong Size { get; private set; }
+ public ulong EndAddress => Address + Size;
+ public PrivateMemoryAllocation PrivateAllocation { get; private set; }
+
+ public PrivateMapping(ulong address, ulong size, PrivateMemoryAllocation privateAllocation)
+ {
+ if (size == 0)
+ {
+ throw new Exception("huh? size is 0");
+ }
+ Address = address;
+ Size = size;
+ PrivateAllocation = privateAllocation;
+ }
+
+ public PrivateMapping Split(ulong splitAddress)
+ {
+ ulong leftSize = splitAddress - Address;
+ ulong rightSize = EndAddress - splitAddress;
+
+ Debug.Assert(leftSize > 0);
+ Debug.Assert(rightSize > 0);
+
+ (var leftAllocation, PrivateAllocation) = PrivateAllocation.Split(leftSize);
+
+ PrivateMapping left = new(Address, leftSize, leftAllocation);
+
+ Address = splitAddress;
+ Size = rightSize;
+
+ return left;
+ }
+
+ public void Map(AddressSpacePartitionAllocation baseBlock, ulong baseAddress, PrivateMemoryAllocation newAllocation)
+ {
+ baseBlock.MapView(newAllocation.Memory, newAllocation.Offset, Address - baseAddress, Size);
+ PrivateAllocation = newAllocation;
+ }
+
+ public void Unmap(AddressSpacePartitionAllocation baseBlock, ulong baseAddress)
+ {
+ if (PrivateAllocation.IsValid)
+ {
+ baseBlock.UnmapView(PrivateAllocation.Memory, Address - baseAddress, Size);
+ PrivateAllocation.Dispose();
+ }
+
+ PrivateAllocation = default;
+ }
+
+ public void Extend(ulong sizeDelta)
+ {
+ Size += sizeDelta;
+ }
+
+ public int CompareTo(PrivateMapping other)
+ {
+ if (Address < other.Address)
+ {
+ return -1;
+ }
+ else if (Address <= other.EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ }
+
+ private readonly MemoryBlock _backingMemory;
+ private readonly AddressSpacePartitionAllocation _baseMemory;
+ private readonly PrivateMemoryAllocator _privateMemoryAllocator;
+ private readonly IntrusiveRedBlackTree<Mapping> _mappingTree;
+ private readonly IntrusiveRedBlackTree<PrivateMapping> _privateTree;
+
+ private readonly object _treeLock;
+
+ private readonly ulong _hostPageSize;
+
+ private ulong? _firstPagePa;
+ private ulong? _lastPagePa;
+ private ulong _cachedFirstPagePa;
+ private ulong _cachedLastPagePa;
+ private bool _hasBridgeAtEnd;
+ private MemoryPermission _lastPageProtection;
+
+ public ulong Address { get; }
+ public ulong Size { get; }
+ public ulong EndAddress => Address + Size;
+
+ public AddressSpacePartition(AddressSpacePartitionAllocation baseMemory, MemoryBlock backingMemory, ulong address, ulong size)
+ {
+ _privateMemoryAllocator = new PrivateMemoryAllocator(DefaultBlockAlignment, MemoryAllocationFlags.Mirrorable);
+ _mappingTree = new IntrusiveRedBlackTree<Mapping>();
+ _privateTree = new IntrusiveRedBlackTree<PrivateMapping>();
+ _treeLock = new object();
+
+ _mappingTree.Add(new Mapping(address, size, MappingType.None));
+ _privateTree.Add(new PrivateMapping(address, size, default));
+
+ _hostPageSize = MemoryBlock.GetPageSize();
+
+ _backingMemory = backingMemory;
+ _baseMemory = baseMemory;
+
+ _cachedFirstPagePa = ulong.MaxValue;
+ _cachedLastPagePa = ulong.MaxValue;
+ _lastPageProtection = MemoryPermission.ReadAndWrite;
+
+ Address = address;
+ Size = size;
+ }
+
+ public bool IsEmpty()
+ {
+ lock (_treeLock)
+ {
+ Mapping map = _mappingTree.GetNode(new Mapping(Address, Size, MappingType.None));
+
+ return map != null && map.Address == Address && map.Size == Size && map.Type == MappingType.None;
+ }
+ }
+
+ public void Map(ulong va, ulong pa, ulong size)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ if (va == Address)
+ {
+ _firstPagePa = pa;
+ }
+
+ if (va <= EndAddress - GuestPageSize && va + size > EndAddress - GuestPageSize)
+ {
+ _lastPagePa = pa + ((EndAddress - GuestPageSize) - va);
+ }
+
+ lock (_treeLock)
+ {
+ Update(va, pa, size, MappingType.Private);
+ }
+ }
+
+ public void Unmap(ulong va, ulong size)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ if (va == Address)
+ {
+ _firstPagePa = null;
+ }
+
+ if (va <= EndAddress - GuestPageSize && va + size > EndAddress - GuestPageSize)
+ {
+ _lastPagePa = null;
+ }
+
+ lock (_treeLock)
+ {
+ Update(va, 0UL, size, MappingType.None);
+ }
+ }
+
+ public void Reprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ _baseMemory.Reprotect(va - Address, size, protection, false);
+
+ if (va == EndAddress - _hostPageSize)
+ {
+ // Protections at the last page also applies to the bridge, if we have one.
+ // (This is because last page access is always done on the bridge, not on our base mapping,
+ // for the cases where access crosses a page boundary and reaches the non-contiguous next mapping).
+
+ if (_hasBridgeAtEnd)
+ {
+ _baseMemory.Reprotect(Size, size, protection, false);
+ }
+
+ _lastPageProtection = protection;
+ }
+ }
+
+ public IntPtr GetPointer(ulong va, ulong size)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ if (va >= EndAddress - _hostPageSize && _hasBridgeAtEnd)
+ {
+ return _baseMemory.GetPointer(Size + va - (EndAddress - _hostPageSize), size);
+ }
+
+ return _baseMemory.GetPointer(va - Address, size);
+ }
+
+ public void InsertBridgeAtEnd(AddressSpacePartition partitionAfter, Action<ulong, IntPtr, ulong> updatePtCallback)
+ {
+ ulong firstPagePa = partitionAfter._firstPagePa.HasValue ? partitionAfter._firstPagePa.Value : ulong.MaxValue;
+ ulong lastPagePa = _lastPagePa.HasValue ? _lastPagePa.Value : ulong.MaxValue;
+
+ if (firstPagePa != _cachedFirstPagePa || lastPagePa != _cachedLastPagePa)
+ {
+ if (partitionAfter._firstPagePa.HasValue && _lastPagePa.HasValue)
+ {
+ (MemoryBlock firstPageMemory, ulong firstPageOffset) = partitionAfter.GetFirstPageMemoryAndOffset();
+ (MemoryBlock lastPageMemory, ulong lastPageOffset) = GetLastPageMemoryAndOffset();
+
+ _baseMemory.MapView(lastPageMemory, lastPageOffset, Size, _hostPageSize);
+ _baseMemory.MapView(firstPageMemory, firstPageOffset, Size + _hostPageSize, _hostPageSize);
+
+ _baseMemory.Reprotect(Size, _hostPageSize, _lastPageProtection, false);
+
+ updatePtCallback(EndAddress - _hostPageSize, _baseMemory.GetPointer(Size, _hostPageSize), _hostPageSize);
+
+ _hasBridgeAtEnd = true;
+ }
+ else
+ {
+ if (_lastPagePa.HasValue)
+ {
+ (MemoryBlock lastPageMemory, ulong lastPageOffset) = GetLastPageMemoryAndOffset();
+
+ updatePtCallback(EndAddress - _hostPageSize, lastPageMemory.GetPointer(lastPageOffset, _hostPageSize), _hostPageSize);
+ }
+
+ _hasBridgeAtEnd = false;
+ }
+
+ _cachedFirstPagePa = firstPagePa;
+ _cachedLastPagePa = lastPagePa;
+ }
+ }
+
+ public void RemoveBridgeFromEnd(Action<ulong, IntPtr, ulong> updatePtCallback)
+ {
+ if (_lastPagePa.HasValue)
+ {
+ (MemoryBlock lastPageMemory, ulong lastPageOffset) = GetLastPageMemoryAndOffset();
+
+ updatePtCallback(EndAddress - _hostPageSize, lastPageMemory.GetPointer(lastPageOffset, _hostPageSize), _hostPageSize);
+ }
+
+ _cachedFirstPagePa = ulong.MaxValue;
+ _cachedLastPagePa = ulong.MaxValue;
+
+ _hasBridgeAtEnd = false;
+ }
+
+ private (MemoryBlock, ulong) GetFirstPageMemoryAndOffset()
+ {
+ lock (_treeLock)
+ {
+ PrivateMapping map = _privateTree.GetNode(new PrivateMapping(Address, 1UL, default));
+
+ if (map != null && map.PrivateAllocation.IsValid)
+ {
+ return (map.PrivateAllocation.Memory, map.PrivateAllocation.Offset + (Address - map.Address));
+ }
+ }
+
+ return (_backingMemory, _firstPagePa.Value);
+ }
+
+ private (MemoryBlock, ulong) GetLastPageMemoryAndOffset()
+ {
+ lock (_treeLock)
+ {
+ ulong pageAddress = EndAddress - _hostPageSize;
+
+ PrivateMapping map = _privateTree.GetNode(new PrivateMapping(pageAddress, 1UL, default));
+
+ if (map != null && map.PrivateAllocation.IsValid)
+ {
+ return (map.PrivateAllocation.Memory, map.PrivateAllocation.Offset + (pageAddress - map.Address));
+ }
+ }
+
+ return (_backingMemory, _lastPagePa.Value & ~(_hostPageSize - 1));
+ }
+
+ private void Update(ulong va, ulong pa, ulong size, MappingType type)
+ {
+ Mapping map = _mappingTree.GetNode(new Mapping(va, 1UL, MappingType.None));
+
+ Update(map, va, pa, size, type);
+ }
+
+ private Mapping Update(Mapping map, ulong va, ulong pa, ulong size, MappingType type)
+ {
+ ulong endAddress = va + size;
+
+ for (; map != null; map = map.Successor)
+ {
+ if (map.Address < va)
+ {
+ _mappingTree.Add(map.Split(va));
+ }
+
+ if (map.EndAddress > endAddress)
+ {
+ Mapping newMap = map.Split(endAddress);
+ _mappingTree.Add(newMap);
+ map = newMap;
+ }
+
+ switch (type)
+ {
+ case MappingType.None:
+ ulong alignment = MemoryBlock.GetPageSize();
+
+ bool unmappedBefore = map.Predecessor == null ||
+ (map.Predecessor.Type == MappingType.None && map.Predecessor.Address <= BitUtils.AlignDown(va, alignment));
+
+ bool unmappedAfter = map.Successor == null ||
+ (map.Successor.Type == MappingType.None && map.Successor.EndAddress >= BitUtils.AlignUp(endAddress, alignment));
+
+ UnmapPrivate(va, size, unmappedBefore, unmappedAfter);
+ break;
+ case MappingType.Private:
+ MapPrivate(va, size);
+ break;
+ }
+
+ map.UpdateState(type);
+ map = TryCoalesce(map);
+
+ if (map.EndAddress >= endAddress)
+ {
+ break;
+ }
+ }
+
+ return map;
+ }
+
+ private Mapping TryCoalesce(Mapping map)
+ {
+ Mapping previousMap = map.Predecessor;
+ Mapping nextMap = map.Successor;
+
+ if (previousMap != null && CanCoalesce(previousMap, map))
+ {
+ previousMap.Extend(map.Size);
+ _mappingTree.Remove(map);
+ map = previousMap;
+ }
+
+ if (nextMap != null && CanCoalesce(map, nextMap))
+ {
+ map.Extend(nextMap.Size);
+ _mappingTree.Remove(nextMap);
+ }
+
+ return map;
+ }
+
+ private static bool CanCoalesce(Mapping left, Mapping right)
+ {
+ return left.Type == right.Type;
+ }
+
+ private void MapPrivate(ulong va, ulong size)
+ {
+ ulong endAddress = va + size;
+
+ ulong alignment = MemoryBlock.GetPageSize();
+
+ // Expand the range outwards based on page size to ensure that at least the requested region is mapped.
+ ulong vaAligned = BitUtils.AlignDown(va, alignment);
+ ulong endAddressAligned = BitUtils.AlignUp(endAddress, alignment);
+
+ PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, 1UL, default));
+
+ for (; map != null; map = map.Successor)
+ {
+ if (!map.PrivateAllocation.IsValid)
+ {
+ if (map.Address < vaAligned)
+ {
+ _privateTree.Add(map.Split(vaAligned));
+ }
+
+ if (map.EndAddress > endAddressAligned)
+ {
+ PrivateMapping newMap = map.Split(endAddressAligned);
+ _privateTree.Add(newMap);
+ map = newMap;
+ }
+
+ map.Map(_baseMemory, Address, _privateMemoryAllocator.Allocate(map.Size, MemoryBlock.GetPageSize()));
+ }
+
+ if (map.EndAddress >= endAddressAligned)
+ {
+ break;
+ }
+ }
+ }
+
+ private void UnmapPrivate(ulong va, ulong size, bool unmappedBefore, bool unmappedAfter)
+ {
+ ulong endAddress = va + size;
+
+ ulong alignment = MemoryBlock.GetPageSize();
+
+ // If the adjacent mappings are unmapped, expand the range outwards,
+ // otherwise shrink it inwards. We must ensure we won't unmap pages that might still be in use.
+ ulong vaAligned = unmappedBefore ? BitUtils.AlignDown(va, alignment) : BitUtils.AlignUp(va, alignment);
+ ulong endAddressAligned = unmappedAfter ? BitUtils.AlignUp(endAddress, alignment) : BitUtils.AlignDown(endAddress, alignment);
+
+ if (endAddressAligned <= vaAligned)
+ {
+ return;
+ }
+
+ PrivateMapping map = _privateTree.GetNode(new PrivateMapping(vaAligned, 1UL, default));
+
+ for (; map != null; map = map.Successor)
+ {
+ if (map.PrivateAllocation.IsValid)
+ {
+ if (map.Address < vaAligned)
+ {
+ _privateTree.Add(map.Split(vaAligned));
+ }
+
+ if (map.EndAddress > endAddressAligned)
+ {
+ PrivateMapping newMap = map.Split(endAddressAligned);
+ _privateTree.Add(newMap);
+ map = newMap;
+ }
+
+ map.Unmap(_baseMemory, Address);
+ map = TryCoalesce(map);
+ }
+
+ if (map.EndAddress >= endAddressAligned)
+ {
+ break;
+ }
+ }
+ }
+
+ private PrivateMapping TryCoalesce(PrivateMapping map)
+ {
+ PrivateMapping previousMap = map.Predecessor;
+ PrivateMapping nextMap = map.Successor;
+
+ if (previousMap != null && CanCoalesce(previousMap, map))
+ {
+ previousMap.Extend(map.Size);
+ _privateTree.Remove(map);
+ map = previousMap;
+ }
+
+ if (nextMap != null && CanCoalesce(map, nextMap))
+ {
+ map.Extend(nextMap.Size);
+ _privateTree.Remove(nextMap);
+ }
+
+ return map;
+ }
+
+ private static bool CanCoalesce(PrivateMapping left, PrivateMapping right)
+ {
+ return !left.PrivateAllocation.IsValid && !right.PrivateAllocation.IsValid;
+ }
+
+ public PrivateRange GetFirstPrivateAllocation(ulong va, ulong size, out ulong nextVa)
+ {
+ lock (_treeLock)
+ {
+ PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, 1UL, default));
+
+ nextVa = map.EndAddress;
+
+ if (map != null && map.PrivateAllocation.IsValid)
+ {
+ ulong startOffset = va - map.Address;
+
+ return new(
+ map.PrivateAllocation.Memory,
+ map.PrivateAllocation.Offset + startOffset,
+ Math.Min(map.PrivateAllocation.Size - startOffset, size));
+ }
+ }
+
+ return PrivateRange.Empty;
+ }
+
+ public bool HasPrivateAllocation(ulong va, ulong size)
+ {
+ lock (_treeLock)
+ {
+ PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, size, default));
+
+ return map != null && map.PrivateAllocation.IsValid;
+ }
+ }
+
+ public void Dispose()
+ {
+ GC.SuppressFinalize(this);
+
+ _privateMemoryAllocator?.Dispose();
+ _baseMemory.Dispose();
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/AddressSpacePartitionAllocator.cs b/src/Ryujinx.Cpu/Jit/AddressSpacePartitionAllocator.cs
new file mode 100644
index 000000000..f09f4e744
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/AddressSpacePartitionAllocator.cs
@@ -0,0 +1,174 @@
+using Ryujinx.Common.Collections;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Tracking;
+using System;
+
+namespace Ryujinx.Cpu.Jit
+{
+ readonly struct AddressSpacePartitionAllocation : IDisposable
+ {
+ private readonly AddressSpacePartitionAllocator _owner;
+ private readonly PrivateMemoryAllocatorImpl<AddressSpacePartitionAllocator.Block>.Allocation _allocation;
+
+ public IntPtr Pointer => (IntPtr)((ulong)_allocation.Block.Memory.Pointer + _allocation.Offset);
+
+ public AddressSpacePartitionAllocation(
+ AddressSpacePartitionAllocator owner,
+ PrivateMemoryAllocatorImpl<AddressSpacePartitionAllocator.Block>.Allocation allocation)
+ {
+ _owner = owner;
+ _allocation = allocation;
+ }
+
+ public void RegisterMapping(ulong va, ulong endVa, int bridgeSize)
+ {
+ _allocation.Block.AddMapping(_allocation.Offset, _allocation.Size, va, endVa, bridgeSize);
+ }
+
+ public void MapView(MemoryBlock srcBlock, ulong srcOffset, ulong dstOffset, ulong size)
+ {
+ _allocation.Block.Memory.MapView(srcBlock, srcOffset, _allocation.Offset + dstOffset, size);
+ }
+
+ public void UnmapView(MemoryBlock srcBlock, ulong offset, ulong size)
+ {
+ _allocation.Block.Memory.UnmapView(srcBlock, _allocation.Offset + offset, size);
+ }
+
+ public void Reprotect(ulong offset, ulong size, MemoryPermission permission, bool throwOnFail)
+ {
+ _allocation.Block.Memory.Reprotect(_allocation.Offset + offset, size, permission, throwOnFail);
+ }
+
+ public IntPtr GetPointer(ulong offset, ulong size)
+ {
+ return _allocation.Block.Memory.GetPointer(_allocation.Offset + offset, size);
+ }
+
+ public void Dispose()
+ {
+ lock (_owner.Lock)
+ {
+ _allocation.Block.RemoveMapping(_allocation.Offset, _allocation.Size);
+ _owner.Free(_allocation.Block, _allocation.Offset, _allocation.Size);
+ }
+ }
+ }
+
+ class AddressSpacePartitionAllocator : PrivateMemoryAllocatorImpl<AddressSpacePartitionAllocator.Block>
+ {
+ private const ulong DefaultBlockAlignment = 1UL << 32; // 4GB
+
+ public class Block : PrivateMemoryAllocator.Block
+ {
+ private readonly MemoryTracking _tracking;
+ private readonly MemoryEhMeilleure _memoryEh;
+
+ private class Mapping : IntrusiveRedBlackTreeNode<Mapping>, IComparable<Mapping>
+ {
+ public ulong Address { get; }
+ public ulong Size { get; }
+ public ulong EndAddress => Address + Size;
+ public ulong Va { get; }
+ public ulong EndVa { get; }
+ public int BridgeSize { get; }
+
+ public Mapping(ulong address, ulong size, ulong va, ulong endVa, int bridgeSize)
+ {
+ Address = address;
+ Size = size;
+ Va = va;
+ EndVa = endVa;
+ BridgeSize = bridgeSize;
+ }
+
+ public int CompareTo(Mapping other)
+ {
+ if (Address < other.Address)
+ {
+ return -1;
+ }
+ else if (Address <= other.EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ }
+
+ private readonly IntrusiveRedBlackTree<Mapping> _mappingTree;
+
+ public Block(MemoryTracking tracking, MemoryBlock memory, ulong size) : base(memory, size)
+ {
+ _tracking = tracking;
+ _memoryEh = new(memory, null, tracking, VirtualMemoryEvent);
+ _mappingTree = new();
+ }
+
+ public void AddMapping(ulong offset, ulong size, ulong va, ulong endVa, int bridgeSize)
+ {
+ _mappingTree.Add(new(offset, size, va, endVa, bridgeSize));
+ }
+
+ public void RemoveMapping(ulong offset, ulong size)
+ {
+ _mappingTree.Remove(_mappingTree.GetNode(new Mapping(offset, size, 0, 0, 0)));
+ }
+
+ private bool VirtualMemoryEvent(ulong address, ulong size, bool write)
+ {
+ Mapping map = _mappingTree.GetNode(new Mapping(address, size, 0, 0, 0));
+
+ if (map == null)
+ {
+ return false;
+ }
+
+ address -= map.Address;
+
+ if (address >= (map.EndVa - map.Va))
+ {
+ address -= (ulong)(map.BridgeSize / 2);
+ }
+
+ return _tracking.VirtualMemoryEvent(map.Va + address, size, write);
+ }
+
+ public override void Destroy()
+ {
+ _memoryEh.Dispose();
+
+ base.Destroy();
+ }
+ }
+
+ private readonly MemoryTracking _tracking;
+
+ public object Lock { get; }
+
+ public AddressSpacePartitionAllocator(MemoryTracking tracking) : base(DefaultBlockAlignment, MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible)
+ {
+ _tracking = tracking;
+ Lock = new();
+ }
+
+ public AddressSpacePartitionAllocation Allocate(ulong va, ulong size, int bridgeSize)
+ {
+ lock (Lock)
+ {
+ AddressSpacePartitionAllocation allocation = new(this, Allocate(size + (ulong)bridgeSize, MemoryBlock.GetPageSize(), CreateBlock));
+ allocation.RegisterMapping(va, va + size, bridgeSize);
+
+ return allocation;
+ }
+ }
+
+ private Block CreateBlock(MemoryBlock memory, ulong size)
+ {
+ return new Block(_tracking, memory, size);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Ryujinx.Cpu/Jit/AddressSpacePartitioned.cs b/src/Ryujinx.Cpu/Jit/AddressSpacePartitioned.cs
new file mode 100644
index 000000000..7ef008690
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/AddressSpacePartitioned.cs
@@ -0,0 +1,362 @@
+using Ryujinx.Common;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Ryujinx.Cpu.Jit
+{
+ class AddressSpacePartitioned : IDisposable
+ {
+ private const int PartitionBits = 25;
+ private const ulong PartitionSize = 1UL << PartitionBits;
+
+ private readonly MemoryBlock _backingMemory;
+ private readonly List<AddressSpacePartition> _partitions;
+ private readonly AddressSpacePartitionAllocator _asAllocator;
+ private readonly Action<ulong, IntPtr, ulong> _updatePtCallback;
+
+ public AddressSpacePartitioned(MemoryTracking tracking, MemoryBlock backingMemory, Action<ulong, IntPtr, ulong> updatePtCallback)
+ {
+ _backingMemory = backingMemory;
+ _partitions = new();
+ _asAllocator = new(tracking);
+ _updatePtCallback = updatePtCallback;
+ }
+
+ public void Map(ulong va, ulong pa, ulong size)
+ {
+ EnsurePartitions(va, size);
+
+ ulong endVa = va + size;
+
+ while (va < endVa)
+ {
+ int partitionIndex = FindPartitionIndex(va);
+ AddressSpacePartition partition = _partitions[partitionIndex];
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ partition.Map(clampedVa, pa, clampedEndVa - clampedVa);
+
+ ulong currentSize = clampedEndVa - clampedVa;
+
+ va += currentSize;
+ pa += currentSize;
+
+ InsertBridgeIfNeeded(partitionIndex);
+ }
+ }
+
+ public void Unmap(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+
+ while (va < endVa)
+ {
+ int partitionIndex = FindPartitionIndex(va);
+ AddressSpacePartition partition = _partitions[partitionIndex];
+
+ if (partition == null)
+ {
+ va += PartitionSize - (va & (PartitionSize - 1));
+
+ continue;
+ }
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ partition.Unmap(clampedVa, clampedEndVa - clampedVa);
+
+ va += clampedEndVa - clampedVa;
+
+ RemoveBridgeIfNeeded(partitionIndex);
+
+ if (partition.IsEmpty())
+ {
+ lock (_partitions)
+ {
+ _partitions.Remove(partition);
+ partition.Dispose();
+ }
+ }
+ }
+ }
+
+ public void Reprotect(ulong va, ulong size, MemoryPermission protection, MemoryTracking tracking)
+ {
+ ulong endVa = va + size;
+
+ while (va < endVa)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ if (partition == null)
+ {
+ va += PartitionSize - (va & (PartitionSize - 1));
+
+ continue;
+ }
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ partition.Reprotect(clampedVa, clampedEndVa - clampedVa, protection);
+
+ va += clampedEndVa - clampedVa;
+ }
+ }
+
+ public PrivateRange GetFirstPrivateAllocation(ulong va, ulong size, out ulong nextVa)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ if (partition == null)
+ {
+ nextVa = (va & ~(PartitionSize - 1)) + PartitionSize;
+
+ return PrivateRange.Empty;
+ }
+
+ return partition.GetFirstPrivateAllocation(va, size, out nextVa);
+ }
+
+ public bool HasAnyPrivateAllocation(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+
+ while (va < endVa)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ if (partition == null)
+ {
+ va += PartitionSize - (va & (PartitionSize - 1));
+
+ continue;
+ }
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ if (partition.HasPrivateAllocation(clampedVa, clampedEndVa - clampedVa))
+ {
+ return true;
+ }
+
+ va += clampedEndVa - clampedVa;
+ }
+
+ return false;
+ }
+
+ private void InsertBridgeIfNeeded(int partitionIndex)
+ {
+ if (partitionIndex > 0 && _partitions[partitionIndex - 1].EndAddress == _partitions[partitionIndex].Address)
+ {
+ _partitions[partitionIndex - 1].InsertBridgeAtEnd(_partitions[partitionIndex], _updatePtCallback);
+ }
+
+ if (partitionIndex + 1 < _partitions.Count && _partitions[partitionIndex].EndAddress == _partitions[partitionIndex + 1].Address)
+ {
+ _partitions[partitionIndex].InsertBridgeAtEnd(_partitions[partitionIndex + 1], _updatePtCallback);
+ }
+ }
+
+ private void RemoveBridgeIfNeeded(int partitionIndex)
+ {
+ if (partitionIndex > 0 && _partitions[partitionIndex - 1].EndAddress == _partitions[partitionIndex].Address)
+ {
+ _partitions[partitionIndex - 1].InsertBridgeAtEnd(_partitions[partitionIndex], _updatePtCallback);
+ }
+
+ if (partitionIndex + 1 < _partitions.Count && _partitions[partitionIndex].EndAddress == _partitions[partitionIndex + 1].Address)
+ {
+ _partitions[partitionIndex].InsertBridgeAtEnd(_partitions[partitionIndex + 1], _updatePtCallback);
+ }
+ else
+ {
+ _partitions[partitionIndex].RemoveBridgeFromEnd(_updatePtCallback);
+ }
+ }
+
+ public IntPtr GetPointer(ulong va, ulong size)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ return partition.GetPointer(va, size);
+ }
+
+ private static (ulong, ulong) ClampRange(AddressSpacePartition partition, ulong va, ulong endVa)
+ {
+ if (va < partition.Address)
+ {
+ va = partition.Address;
+ }
+
+ if (endVa > partition.EndAddress)
+ {
+ endVa = partition.EndAddress;
+ }
+
+ return (va, endVa);
+ }
+
+ private void EnsurePartitions(ulong va, ulong size)
+ {
+ lock (_partitions)
+ {
+ EnsurePartitionsForRange(va, size);
+ }
+ }
+
+ private AddressSpacePartition FindPartition(ulong va)
+ {
+ lock (_partitions)
+ {
+ int index = FindPartitionIndex(va);
+ if (index >= 0)
+ {
+ return _partitions[index];
+ }
+ }
+
+ return null;
+ }
+
+ private int FindPartitionIndex(ulong va)
+ {
+ lock (_partitions)
+ {
+ int left = 0;
+ int middle = 0;
+ int right = _partitions.Count - 1;
+
+ while (left <= right)
+ {
+ middle = left + ((right - left) >> 1);
+
+ AddressSpacePartition partition = _partitions[middle];
+
+ if (partition.Address <= va && partition.EndAddress > va)
+ {
+ return middle;
+ }
+
+ if (partition.Address >= va)
+ {
+ right = middle - 1;
+ }
+ else
+ {
+ left = middle + 1;
+ }
+ }
+ }
+
+ return -1;
+ }
+
+ private void EnsurePartitionsForRange(ulong va, ulong size)
+ {
+ ulong endVa = BitUtils.AlignUp(va + size, PartitionSize);
+ va = BitUtils.AlignDown(va, PartitionSize);
+
+ for (int i = 0; i < _partitions.Count && va < endVa; i++)
+ {
+ AddressSpacePartition partition = _partitions[i];
+
+ if (partition.Address <= va && partition.EndAddress > va)
+ {
+ if (partition.EndAddress >= endVa)
+ {
+ // Fully mapped already.
+ va = endVa;
+
+ break;
+ }
+
+ ulong gapSize;
+
+ if (i + 1 < _partitions.Count)
+ {
+ AddressSpacePartition nextPartition = _partitions[i + 1];
+
+ if (partition.EndAddress == nextPartition.Address)
+ {
+ va = partition.EndAddress;
+
+ continue;
+ }
+
+ gapSize = Math.Min(endVa, nextPartition.Address) - partition.EndAddress;
+ }
+ else
+ {
+ gapSize = endVa - partition.EndAddress;
+ }
+
+ _partitions.Insert(i + 1, new(CreateAsPartitionAllocation(partition.EndAddress, gapSize), _backingMemory, partition.EndAddress, gapSize));
+ va = partition.EndAddress + gapSize;
+ i++;
+ }
+ else if (partition.EndAddress > va)
+ {
+ Debug.Assert(partition.Address > va);
+
+ ulong gapSize;
+
+ if (partition.Address < endVa)
+ {
+ gapSize = partition.Address - va;
+ }
+ else
+ {
+ gapSize = endVa - va;
+ }
+
+ _partitions.Insert(i, new(CreateAsPartitionAllocation(va, gapSize), _backingMemory, va, gapSize));
+ va = Math.Min(partition.EndAddress, endVa);
+ i++;
+ }
+ }
+
+ if (va < endVa)
+ {
+ _partitions.Add(new(CreateAsPartitionAllocation(va, endVa - va), _backingMemory, va, endVa - va));
+ }
+
+ for (int i = 1; i < _partitions.Count; i++)
+ {
+ Debug.Assert(_partitions[i].Address > _partitions[i - 1].Address);
+ Debug.Assert(_partitions[i].EndAddress > _partitions[i - 1].EndAddress);
+ }
+ }
+
+ private AddressSpacePartitionAllocation CreateAsPartitionAllocation(ulong va, ulong size)
+ {
+ ulong bridgeSize = MemoryBlock.GetPageSize() * 2;
+
+ return _asAllocator.Allocate(va, size, (int)bridgeSize);
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ foreach (AddressSpacePartition partition in _partitions)
+ {
+ partition.Dispose();
+ }
+
+ _partitions.Clear();
+ _asAllocator.Dispose();
+ }
+ }
+
+ public void Dispose()
+ {
+ Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
index 2b315e841..6d32787ac 100644
--- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
@@ -1,4 +1,4 @@
-using ARMeilleure.Memory;
+using ARMeilleure.Memory;
using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
new file mode 100644
index 000000000..70175ce72
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
@@ -0,0 +1,1009 @@
+using ARMeilleure.Memory;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Range;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+namespace Ryujinx.Cpu.Jit
+{
+ /// <summary>
+ /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
+ /// </summary>
+ public sealed class MemoryManagerHostTracked : MemoryManagerBase, IWritableBlock, IMemoryManager, IVirtualMemoryManagerTracked
+ {
+ public const int PageBits = 12;
+ public const int PageSize = 1 << PageBits;
+ public const int PageMask = PageSize - 1;
+
+ public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
+ public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
+
+ private enum HostMappedPtBits : ulong
+ {
+ Unmapped = 0,
+ Mapped,
+ WriteTracked,
+ ReadWriteTracked,
+
+ MappedReplicated = 0x5555555555555555,
+ WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
+ ReadWriteTrackedReplicated = ulong.MaxValue
+ }
+
+ private readonly InvalidAccessHandler _invalidAccessHandler;
+
+ private readonly MemoryBlock _backingMemory;
+ private readonly PageTable<ulong> _pageTable;
+
+ private readonly ulong[] _pageBitmap;
+
+ public int AddressSpaceBits { get; }
+
+ public MemoryTracking Tracking { get; private set; }
+
+ private const int PteSize = 8;
+
+ private readonly AddressSpacePartitioned _addressSpace;
+
+ public ulong AddressSpaceSize { get; }
+
+ private readonly MemoryBlock _flatPageTable;
+
+ /// <inheritdoc/>
+ public bool Supports4KBPages => false;
+
+ public IntPtr PageTablePointer => _flatPageTable.Pointer;
+
+ public MemoryManagerType Type => MemoryManagerType.HostTracked;
+
+ public event Action<ulong, ulong> UnmapEvent;
+
+ /// <summary>
+ /// Creates a new instance of the host mapped memory manager.
+ /// </summary>
+ /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
+ /// <param name="addressSpaceSize">Size of the address space</param>
+ /// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
+ public MemoryManagerHostTracked(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler)
+ {
+ Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
+
+ _backingMemory = backingMemory;
+ _pageTable = new PageTable<ulong>();
+ _invalidAccessHandler = invalidAccessHandler;
+ _addressSpace = new(Tracking, backingMemory, UpdatePt);
+ AddressSpaceSize = addressSpaceSize;
+
+ ulong asSize = PageSize;
+ int asBits = PageBits;
+
+ while (asSize < AddressSpaceSize)
+ {
+ asSize <<= 1;
+ asBits++;
+ }
+
+ AddressSpaceBits = asBits;
+
+ _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
+ _flatPageTable = new MemoryBlock((asSize / PageSize) * PteSize);
+ }
+
+ /// <inheritdoc/>
+ public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ if (flags.HasFlag(MemoryMapFlags.Private))
+ {
+ _addressSpace.Map(va, pa, size);
+ }
+
+ AddMapping(va, size);
+ PtMap(va, pa, size, flags.HasFlag(MemoryMapFlags.Private));
+
+ Tracking.Map(va, size);
+ }
+
+ private void PtMap(ulong va, ulong pa, ulong size, bool privateMap)
+ {
+ while (size != 0)
+ {
+ _pageTable.Map(va, pa);
+
+ if (privateMap)
+ {
+ _flatPageTable.Write((va / PageSize) * PteSize, (ulong)_addressSpace.GetPointer(va, PageSize) - va);
+ }
+ else
+ {
+ _flatPageTable.Write((va / PageSize) * PteSize, (ulong)_backingMemory.GetPointer(pa, PageSize) - va);
+ }
+
+ va += PageSize;
+ pa += PageSize;
+ size -= PageSize;
+ }
+ }
+
+ private void UpdatePt(ulong va, IntPtr ptr, ulong size)
+ {
+ ulong remainingSize = size;
+ while (remainingSize != 0)
+ {
+ _flatPageTable.Write((va / PageSize) * PteSize, (ulong)ptr - va);
+
+ va += PageSize;
+ ptr += PageSize;
+ remainingSize -= PageSize;
+ }
+ }
+
+ /// <inheritdoc/>
+ public void MapForeign(ulong va, nuint hostPointer, ulong size)
+ {
+ throw new NotSupportedException();
+ }
+
+ /// <inheritdoc/>
+ public void Unmap(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ _addressSpace.Unmap(va, size);
+
+ UnmapEvent?.Invoke(va, size);
+ Tracking.Unmap(va, size);
+
+ RemoveMapping(va, size);
+ PtUnmap(va, size);
+ }
+
+ private void PtUnmap(ulong va, ulong size)
+ {
+ while (size != 0)
+ {
+ _pageTable.Unmap(va);
+ _flatPageTable.Write((va / PageSize) * PteSize, 0UL);
+
+ va += PageSize;
+ size -= PageSize;
+ }
+ }
+
+ /// <summary>
+ /// Checks if the virtual address is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address</param>
+ /// <returns>True if the virtual address is part of the addressable space</returns>
+ private bool ValidateAddress(ulong va)
+ {
+ return va < AddressSpaceSize;
+ }
+
+ /// <summary>
+ /// Checks if the combination of virtual address and size is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <returns>True if the combination of virtual address and size is part of the addressable space</returns>
+ private bool ValidateAddressAndSize(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+ return endVa >= va && endVa >= size && endVa <= AddressSpaceSize;
+ }
+
+ /// <summary>
+ /// Ensures the combination of virtual address and size is part of the addressable space.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <exception cref="InvalidMemoryRegionException">Throw when the memory region specified outside the addressable space</exception>
+ private void AssertValidAddressAndSize(ulong va, ulong size)
+ {
+ if (!ValidateAddressAndSize(va, size))
+ {
+ throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+
+ public T Read<T>(ulong va) where T : unmanaged
+ {
+ return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
+ }
+
+ public T ReadTracked<T>(ulong va) where T : unmanaged
+ {
+ try
+ {
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
+
+ return Read<T>(va);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return default;
+ }
+ }
+
+ public void Read(ulong va, Span<byte> data)
+ {
+ ReadImpl(va, data);
+ }
+
+ public void Write<T>(ulong va, T value) where T : unmanaged
+ {
+ Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
+ }
+
+ public void Write(ulong va, ReadOnlySpan<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ SignalMemoryTracking(va, (ulong)data.Length, true);
+
+ WriteImpl(va, data);
+ }
+
+ public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ WriteImpl(va, data);
+ }
+
+ public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return false;
+ }
+
+ SignalMemoryTracking(va, (ulong)data.Length, false);
+
+ if (TryGetVirtualContiguous(va, data.Length, out MemoryBlock memoryBlock, out ulong offset))
+ {
+ var target = memoryBlock.GetSpan(offset, data.Length);
+
+ bool changed = !data.SequenceEqual(target);
+
+ if (changed)
+ {
+ data.CopyTo(target);
+ }
+
+ return changed;
+ }
+ else
+ {
+ WriteImpl(va, data);
+
+ return true;
+ }
+ }
+
+ private void WriteImpl(ulong va, ReadOnlySpan<byte> data)
+ {
+ try
+ {
+ AssertValidAddressAndSize(va, (ulong)data.Length);
+
+ ulong endVa = va + (ulong)data.Length;
+ int offset = 0;
+
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
+
+ data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize));
+
+ va += copySize;
+ offset += (int)copySize;
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return ReadOnlySpan<byte>.Empty;
+ }
+
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, false);
+ }
+
+ if (TryGetVirtualContiguous(va, size, out MemoryBlock memoryBlock, out ulong offset))
+ {
+ return memoryBlock.GetSpan(offset, size);
+ }
+ else
+ {
+ Span<byte> data = new byte[size];
+
+ ReadImpl(va, data);
+
+ return data;
+ }
+ }
+
+ public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return new WritableRegion(null, va, Memory<byte>.Empty);
+ }
+
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, true);
+ }
+
+ if (TryGetVirtualContiguous(va, size, out MemoryBlock memoryBlock, out ulong offset))
+ {
+ return new WritableRegion(null, va, memoryBlock.GetMemory(offset, size));
+ }
+ else
+ {
+ Memory<byte> memory = new byte[size];
+
+ ReadImpl(va, memory.Span);
+
+ return new WritableRegion(this, va, memory);
+ }
+ }
+
+ public ref T GetRef<T>(ulong va) where T : unmanaged
+ {
+ if (!TryGetVirtualContiguous(va, Unsafe.SizeOf<T>(), out MemoryBlock memory, out ulong offset))
+ {
+ ThrowMemoryNotContiguous();
+ }
+
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
+
+ return ref memory.GetRef<T>(offset);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public bool IsMapped(ulong va)
+ {
+ return ValidateAddress(va) && IsMappedImpl(va);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsMappedImpl(ulong va)
+ {
+ ulong page = va >> PageBits;
+
+ int bit = (int)((page & 31) << 1);
+
+ int pageIndex = (int)(page >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte = Volatile.Read(ref pageRef);
+
+ return ((pte >> bit) & 3) != 0;
+ }
+
+ public bool IsRangeMapped(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ return IsRangeMappedImpl(va, size);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
+ {
+ startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
+ endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
+
+ pageIndex = (int)(pageStart >> PageToPteShift);
+ pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
+ }
+
+ private bool IsRangeMappedImpl(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+
+ if (pages == 1)
+ {
+ return IsMappedImpl(va);
+ }
+
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ // Check if either bit in each 2 bit page entry is set.
+ // OR the block with itself shifted down by 1, and check the first bit of each entry.
+
+ ulong mask = BlockMappedMask & startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+ ulong pte = Volatile.Read(ref pageRef);
+
+ pte |= pte >> 1;
+ if ((pte & mask) != mask)
+ {
+ return false;
+ }
+
+ mask = BlockMappedMask;
+ }
+
+ return true;
+ }
+
+ private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
+
+ private bool TryGetVirtualContiguous(ulong va, int size, out MemoryBlock memory, out ulong offset)
+ {
+ if (_addressSpace.HasAnyPrivateAllocation(va, (ulong)size))
+ {
+ // If we have a private allocation overlapping the range,
+ // this the access is only considered contiguous if it covers the entire range.
+
+ PrivateRange range = _addressSpace.GetFirstPrivateAllocation(va, (ulong)size, out _);
+
+ if (range.Memory != null && range.Size == (ulong)size)
+ {
+ memory = range.Memory;
+ offset = range.Offset;
+
+ return true;
+ }
+
+ memory = null;
+ offset = 0;
+
+ return false;
+ }
+
+ memory = _backingMemory;
+ offset = GetPhysicalAddressInternal(va);
+
+ return IsPhysicalContiguous(va, size);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsPhysicalContiguousAndMapped(ulong va, int size) => IsPhysicalContiguous(va, size) && IsMapped(va);
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsPhysicalContiguous(ulong va, int size)
+ {
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
+ {
+ return false;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return false;
+ }
+
+ if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+ {
+ return false;
+ }
+
+ va += PageSize;
+ }
+
+ return true;
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private ulong GetContiguousSize(ulong va, ulong size)
+ {
+ ulong contiguousSize = PageSize - (va & PageMask);
+
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
+ {
+ return contiguousSize;
+ }
+
+ int pages = GetPagesCount(va, size, out va);
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return contiguousSize;
+ }
+
+ if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+ {
+ return contiguousSize;
+ }
+
+ va += PageSize;
+ contiguousSize += PageSize;
+ }
+
+ return Math.Min(contiguousSize, size);
+ }
+
+ private (MemoryBlock, ulong, ulong) GetMemoryOffsetAndSize(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+
+ PrivateRange privateRange = _addressSpace.GetFirstPrivateAllocation(va, size, out ulong nextVa);
+
+ if (privateRange.Memory != null)
+ {
+ return (privateRange.Memory, privateRange.Offset, privateRange.Size);
+ }
+
+ ulong physSize = GetContiguousSize(va, Math.Min(size, nextVa - va));
+
+ return new(_backingMemory, GetPhysicalAddressChecked(va), physSize);
+ }
+
+ public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
+ {
+ if (!ValidateAddressAndSize(va, size))
+ {
+ return null;
+ }
+
+ var regions = new List<HostMemoryRange>();
+ ulong endVa = va + size;
+
+ try
+ {
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong rangeSize) = GetMemoryOffsetAndSize(va, endVa - va);
+
+ regions.Add(new((UIntPtr)memory.GetPointer(rangeOffset, rangeSize), rangeSize));
+
+ va += rangeSize;
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ return null;
+ }
+
+ return regions;
+ }
+
+ public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
+ {
+ if (size == 0)
+ {
+ return Enumerable.Empty<MemoryRange>();
+ }
+
+ return GetPhysicalRegionsImpl(va, size);
+ }
+
+ private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
+ {
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
+ {
+ return null;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ var regions = new List<MemoryRange>();
+
+ ulong regionStart = GetPhysicalAddressInternal(va);
+ ulong regionSize = PageSize;
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return null;
+ }
+
+ ulong newPa = GetPhysicalAddressInternal(va + PageSize);
+
+ if (GetPhysicalAddressInternal(va) + PageSize != newPa)
+ {
+ regions.Add(new MemoryRange(regionStart, regionSize));
+ regionStart = newPa;
+ regionSize = 0;
+ }
+
+ va += PageSize;
+ regionSize += PageSize;
+ }
+
+ regions.Add(new MemoryRange(regionStart, regionSize));
+
+ return regions;
+ }
+
+ private void ReadImpl(ulong va, Span<byte> data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ try
+ {
+ AssertValidAddressAndSize(va, (ulong)data.Length);
+
+ ulong endVa = va + (ulong)data.Length;
+ int offset = 0;
+
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
+
+ memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize));
+
+ va += copySize;
+ offset += (int)copySize;
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ /// <inheritdoc/>
+ /// <remarks>
+ /// This function also validates that the given range is both valid and mapped, and will throw if it is not.
+ /// </remarks>
+ public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ if (precise)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
+ return;
+ }
+
+ // Software table, used for managed memory tracking.
+
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+
+ if (pages == 1)
+ {
+ ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked);
+
+ int bit = (int)((pageStart & 31) << 1);
+
+ int pageIndex = (int)(pageStart >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte = Volatile.Read(ref pageRef);
+ ulong state = ((pte >> bit) & 3);
+
+ if (state >= tag)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ return;
+ }
+ else if (state == 0)
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+ else
+ {
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte = Volatile.Read(ref pageRef);
+ ulong mappedMask = mask & BlockMappedMask;
+
+ ulong mappedPte = pte | (pte >> 1);
+ if ((mappedPte & mappedMask) != mappedMask)
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+
+ pte &= mask;
+ if ((pte & anyTrackingTag) != 0) // Search for any tracking.
+ {
+ // Writes trigger any tracking.
+ // Only trigger tracking from reads if both bits are set on any page.
+ if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ break;
+ }
+ }
+
+ mask = ulong.MaxValue;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Computes the number of pages in a virtual address range.
+ /// </summary>
+ /// <param name="va">Virtual address of the range</param>
+ /// <param name="size">Size of the range</param>
+ /// <param name="startVa">The virtual address of the beginning of the first page</param>
+ /// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private int GetPagesCount(ulong va, ulong size, out ulong startVa)
+ {
+ // WARNING: Always check if ulong does not overflow during the operations.
+ startVa = va & ~(ulong)PageMask;
+ ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
+
+ return (int)(vaSpan / PageSize);
+ }
+
+ public RegionHandle BeginTracking(ulong address, ulong size, int id)
+ {
+ return Tracking.BeginTracking(address, size, id);
+ }
+
+ public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id)
+ {
+ return Tracking.BeginGranularTracking(address, size, handles, granularity, id);
+ }
+
+ public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
+ {
+ return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
+ }
+
+ /// <summary>
+ /// Adds the given address mapping to the page table.
+ /// </summary>
+ /// <param name="va">Virtual memory address</param>
+ /// <param name="size">Size to be mapped</param>
+ private void AddMapping(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte;
+ ulong mappedMask;
+
+ // Map all 2-bit entries that are unmapped.
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+
+ mappedMask = pte | (pte >> 1);
+ mappedMask |= (mappedMask & BlockMappedMask) << 1;
+ mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
+ }
+ while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
+
+ mask = ulong.MaxValue;
+ }
+ }
+
+ /// <summary>
+ /// Removes the given address mapping from the page table.
+ /// </summary>
+ /// <param name="va">Virtual memory address</param>
+ /// <param name="size">Size to be unmapped</param>
+ private void RemoveMapping(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ startMask = ~startMask;
+ endMask = ~endMask;
+
+ ulong mask = startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask |= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+ ulong pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
+
+ mask = 0;
+ }
+ }
+
+ private ulong GetPhysicalAddressChecked(ulong va)
+ {
+ if (!IsMapped(va))
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
+ }
+
+ return GetPhysicalAddressInternal(va);
+ }
+
+ private ulong GetPhysicalAddressInternal(ulong va)
+ {
+ return _pageTable.Read(va) + (va & PageMask);
+ }
+
+ private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
+
+ /// <inheritdoc/>
+ public void Reprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ // TODO
+ }
+
+ /// <inheritdoc/>
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ // Protection is inverted on software pages, since the default value is 0.
+ protection = (~protection) & MemoryPermission.ReadAndWrite;
+
+ int pages = GetPagesCount(va, size, out va);
+ ulong pageStart = va >> PageBits;
+
+ if (pages == 1)
+ {
+ ulong protTag = protection switch
+ {
+ MemoryPermission.None => (ulong)HostMappedPtBits.Mapped,
+ MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked,
+ _ => (ulong)HostMappedPtBits.ReadWriteTracked,
+ };
+
+ int bit = (int)((pageStart & 31) << 1);
+
+ ulong tagMask = 3UL << bit;
+ ulong invTagMask = ~tagMask;
+
+ ulong tag = protTag << bit;
+
+ int pageIndex = (int)(pageStart >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+ }
+ else
+ {
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ ulong protTag = protection switch
+ {
+ MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated,
+ MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated,
+ _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated,
+ };
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte;
+ ulong mappedMask;
+
+ // Change the protection of all 2 bit entries that are mapped.
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+
+ mappedMask = pte | (pte >> 1);
+ mappedMask |= (mappedMask & BlockMappedMask) << 1;
+ mappedMask &= mask; // Only update mapped pages within the given range.
+ }
+ while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
+
+ mask = ulong.MaxValue;
+ }
+ }
+
+ protection = protection switch
+ {
+ MemoryPermission.None => MemoryPermission.ReadAndWrite,
+ MemoryPermission.Write => MemoryPermission.Read,
+ _ => MemoryPermission.None,
+ };
+
+ _addressSpace.Reprotect(va, size, protection, Tracking);
+ }
+
+ /// <summary>
+ /// Disposes of resources used by the memory manager.
+ /// </summary>
+ protected override void Destroy()
+ {
+ _addressSpace.Dispose();
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/MemoryEhMeilleure.cs b/src/Ryujinx.Cpu/MemoryEhMeilleure.cs
index 54e232d9c..d3763c777 100644
--- a/src/Ryujinx.Cpu/MemoryEhMeilleure.cs
+++ b/src/Ryujinx.Cpu/MemoryEhMeilleure.cs
@@ -1,4 +1,4 @@
-using ARMeilleure.Signal;
+using ARMeilleure.Signal;
using Ryujinx.Memory;
using Ryujinx.Memory.Tracking;
using System;
@@ -8,19 +8,21 @@ namespace Ryujinx.Cpu
{
public class MemoryEhMeilleure : IDisposable
{
- private delegate bool TrackingEventDelegate(ulong address, ulong size, bool write);
+ public delegate bool TrackingEventDelegate(ulong address, ulong size, bool write);
private readonly TrackingEventDelegate _trackingEvent;
private readonly ulong _baseAddress;
private readonly ulong _mirrorAddress;
- public MemoryEhMeilleure(MemoryBlock addressSpace, MemoryBlock addressSpaceMirror, MemoryTracking tracking)
+ public MemoryEhMeilleure(MemoryBlock addressSpace, MemoryBlock addressSpaceMirror, MemoryTracking tracking, TrackingEventDelegate trackingEvent = null)
{
_baseAddress = (ulong)addressSpace.Pointer;
+
ulong endAddress = _baseAddress + addressSpace.Size;
- _trackingEvent = tracking.VirtualMemoryEvent;
+ _trackingEvent = trackingEvent ?? tracking.VirtualMemoryEvent;
+
bool added = NativeSignalHandler.AddTrackedRegion((nuint)_baseAddress, (nuint)endAddress, Marshal.GetFunctionPointerForDelegate(_trackingEvent));
if (!added)
@@ -28,7 +30,7 @@ namespace Ryujinx.Cpu
throw new InvalidOperationException("Number of allowed tracked regions exceeded.");
}
- if (OperatingSystem.IsWindows())
+ if (OperatingSystem.IsWindows() && addressSpaceMirror != null)
{
// Add a tracking event with no signal handler for the mirror on Windows.
// The native handler has its own code to check for the partial overlap race when regions are protected by accident,
diff --git a/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs b/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
index ce8e83419..8db74f1e9 100644
--- a/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
+++ b/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
@@ -143,7 +143,7 @@ namespace Ryujinx.Cpu
}
}
- public PrivateMemoryAllocator(int blockAlignment, MemoryAllocationFlags allocationFlags) : base(blockAlignment, allocationFlags)
+ public PrivateMemoryAllocator(ulong blockAlignment, MemoryAllocationFlags allocationFlags) : base(blockAlignment, allocationFlags)
{
}
@@ -180,10 +180,10 @@ namespace Ryujinx.Cpu
private readonly List<T> _blocks;
- private readonly int _blockAlignment;
+ private readonly ulong _blockAlignment;
private readonly MemoryAllocationFlags _allocationFlags;
- public PrivateMemoryAllocatorImpl(int blockAlignment, MemoryAllocationFlags allocationFlags)
+ public PrivateMemoryAllocatorImpl(ulong blockAlignment, MemoryAllocationFlags allocationFlags)
{
_blocks = new List<T>();
_blockAlignment = blockAlignment;
@@ -212,7 +212,7 @@ namespace Ryujinx.Cpu
}
}
- ulong blockAlignedSize = BitUtils.AlignUp(size, (ulong)_blockAlignment);
+ ulong blockAlignedSize = BitUtils.AlignUp(size, _blockAlignment);
var memory = new MemoryBlock(blockAlignedSize, _allocationFlags);
var newBlock = createBlock(memory, blockAlignedSize);
diff --git a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs
index 06b8fd345..6ca600146 100644
--- a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs
+++ b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs
@@ -1,4 +1,4 @@
-using Ryujinx.Common.Configuration;
+using Ryujinx.Common.Configuration;
using Ryujinx.Common.Logging;
using Ryujinx.Cpu;
using Ryujinx.Cpu.AppleHv;
@@ -72,7 +72,7 @@ namespace Ryujinx.HLE.HOS
AddressSpace addressSpace = null;
- if (mode == MemoryManagerMode.HostMapped || mode == MemoryManagerMode.HostMappedUnsafe)
+ if ((mode == MemoryManagerMode.HostMapped || mode == MemoryManagerMode.HostMappedUnsafe) && MemoryBlock.GetPageSize() <= 0x1000)
{
if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, MemoryBlock.GetPageSize() == MemoryManagerHostMapped.PageSize, out addressSpace))
{
@@ -91,13 +91,21 @@ namespace Ryujinx.HLE.HOS
case MemoryManagerMode.HostMapped:
case MemoryManagerMode.HostMappedUnsafe:
- if (addressSpaceSize != addressSpace.AddressSpaceSize)
+ if (addressSpace == null)
{
- Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{addressSpace.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})");
+ var memoryManagerHostTracked = new MemoryManagerHostTracked(context.Memory, addressSpaceSize, invalidAccessHandler);
+ processContext = new ArmProcessContext<MemoryManagerHostTracked>(pid, cpuEngine, _gpu, memoryManagerHostTracked, addressSpaceSize, for64Bit);
}
+ else
+ {
+ if (addressSpaceSize != addressSpace.AddressSpaceSize)
+ {
+ Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{addressSpace.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})");
+ }
- var memoryManagerHostMapped = new MemoryManagerHostMapped(addressSpace, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler);
- processContext = new ArmProcessContext<MemoryManagerHostMapped>(pid, cpuEngine, _gpu, memoryManagerHostMapped, addressSpace.AddressSpaceSize, for64Bit);
+ var memoryManagerHostMapped = new MemoryManagerHostMapped(addressSpace, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler);
+ processContext = new ArmProcessContext<MemoryManagerHostMapped>(pid, cpuEngine, _gpu, memoryManagerHostMapped, addressSpace.AddressSpaceSize, for64Bit);
+ }
break;
default:
diff --git a/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs b/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs
index 543acb7a0..d7b601d1c 100644
--- a/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs
+++ b/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTable.cs
@@ -165,6 +165,29 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
/// <inheritdoc/>
protected override Result MapForeign(IEnumerable<HostMemoryRange> regions, ulong va, ulong size)
{
+ ulong backingStart = (ulong)Context.Memory.Pointer;
+ ulong backingEnd = backingStart + Context.Memory.Size;
+
+ KPageList pageList = new();
+
+ foreach (HostMemoryRange region in regions)
+ {
+ // If the range is inside the physical memory, it is shared and we should increment the page count,
+ // otherwise it is private and we don't need to increment the page count.
+
+ if (region.Address >= backingStart && region.Address < backingEnd)
+ {
+ pageList.AddRange(region.Address - backingStart + DramMemoryMap.DramBase, region.Size / PageSize);
+ }
+ }
+
+ using var scopedPageList = new KScopedPageList(Context.MemoryManager, pageList);
+
+ foreach (var pageNode in pageList)
+ {
+ Context.CommitMemory(pageNode.Address - DramMemoryMap.DramBase, pageNode.PagesCount * PageSize);
+ }
+
ulong offset = 0;
foreach (var region in regions)
@@ -174,6 +197,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory
offset += region.Size;
}
+ scopedPageList.SignalSuccess();
+
return Result.Success;
}
diff --git a/src/Ryujinx.Memory/AddressSpaceManager.cs b/src/Ryujinx.Memory/AddressSpaceManager.cs
index 021d33663..05447ae39 100644
--- a/src/Ryujinx.Memory/AddressSpaceManager.cs
+++ b/src/Ryujinx.Memory/AddressSpaceManager.cs
@@ -293,9 +293,9 @@ namespace Ryujinx.Memory
{
var hostRegion = hostRegions[i];
- if ((ulong)hostRegion.Address >= backingStart && (ulong)hostRegion.Address < backingEnd)
+ if (hostRegion.Address >= backingStart && hostRegion.Address < backingEnd)
{
- regions[count++] = new MemoryRange((ulong)hostRegion.Address - backingStart, hostRegion.Size);
+ regions[count++] = new MemoryRange(hostRegion.Address - backingStart, hostRegion.Size);
}
}