Using the 2020 iPad’s ARMeshAnchor with SceneKit


I’m sure this will change at WWDC, but at the moment there isn’t any support in SceneKit for the new ARMeshAnchor data generated by the 2020 iPad with Lidar.

This is unfortunate as SceneKit’s very handy for quick prototyping but it’s easy enough (even for a SceneKit rookie like myself) to generate some SCNGeometry from the raw data that ARKit provides.

Here’s a drop in file that adds a fromAnchor extension to SCNGeometry

//
//  SCNGeometry+ARMeshAnchor.swift
//
//  Created by Andrew Grant on 4/17/20.
//

import SceneKit
import ARKit

extension SCNGeometry {
    
    /**
     Constructs an SCNGeometry element from an ARMeshAnchor.
     
     Note, the underlying vertex data is owned by the ARMeshAnchor so this geometry becomes invalid when the
     anchor is updated or removed.
    */
    public static func fromAnchor(meshAnchor: ARMeshAnchor) -> SCNGeometry {
        let vertices = meshAnchor.geometry.vertices
        let faces = meshAnchor.geometry.faces
        
        // use the MTL buffer that ARKit gives us
        let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat: vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride)
        
        // but we need to create our own copy of the faces..
        let faceData = Data(bytesNoCopy: faces.buffer.contents(), count: faces.buffer.length, deallocator: .none)
        
        // create the geometry element
        let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
        let geometry = SCNGeometry(sources: [vertexSource], elements: [geometryElement])
        
        // assign a material suitable for default visualization
        let defaultMaterial = SCNMaterial()
        defaultMaterial.fillMode = .lines
        defaultMaterial.diffuse.contents = UIColor(displayP3Red:1, green:1, blue:1, alpha:0.7)
        geometry.materials = [defaultMaterial]
        
        return geometry;
  }
}

And here’s an example of how to use it in your ARSessionDelegate handler (with a PlaneAnchor as a comparison)

    var knownAnchors = Dictionary<UUID, SCNNode>()
    
    func session(_ session: ARSession, didAdd anchors: [ARAnchor]) {
                
        for anchor in anchors {
            var sceneNode : SCNNode?
            
            if let meshAnchor = anchor as? ARMeshAnchor {
                let meshGeo = SCNGeometry.fromAnchor(meshAnchor:meshAnchor)
                sceneNode = SCNNode(geometry:meshGeo)
            }
            else if let planeAnchor = anchor as? ARPlaneAnchor {
             
                let planeGeo = ARSCNPlaneGeometry(device: arScnView.device!)
                planeGeo?.update(from: planeAnchor.geometry)
                planeGeo?.firstMaterial?.fillMode = .lines
                sceneNode = SCNNode(geometry: planeGeo)
            }
            
            if let node = sceneNode {
                node.simdTransform = anchor.transform
                knownAnchors[anchor.identifier] = node
                arSceneView.scene.rootNode.addChildNode(node)
            }
        }
    }
    
    func session(_ session: ARSession, didUpdate anchors: [ARAnchor]) {
        
        for anchor in anchors {
            
            // do we know about this anchor? If so update it
            if let node = knownAnchors[anchor.identifier] {
                if let planeAnchor = anchor as? ARPlaneAnchor {
                    let planeGeometry = node.geometry as! ARSCNPlaneGeometry
                    planeGeometry.update(from: planeAnchor.geometry)
                }
                else if let meshAnchor = anchor as? ARMeshAnchor {
                    // reconstruct it since we don't have an efficient way of updating the underlying data
                    node.geometry = SCNGeometry.fromAnchor(meshAnchor: meshAnchor)
                }
                node.simdTransform = anchor.transform
            }
        }
    }

(Note the geometry returned by fromMesh uses a *reference* to the data in ARMeshAnchor so you must implement didUpdate to reconstruct the geometry and didRemove to remove the node so it no longer renders. If this isn’t done expect a crash or validation error of some type).

A scan of a cat and a mouse


Leave a Reply

Your email address will not be published. Required fields are marked *