从CVImageBuffer保持MTLTexture导致口吃
我使用MTLTexture从CVImageBuffer （从相机和播放器）创build一个MTLTexture ，获取CVMetalTexture ，然后使用CVMetalTextureGetTexture获取MTLTexture 。
我看到的问题是，当我稍后使用Metal渲染纹理时，偶尔会看到video帧呈现不规则（视觉上来回拖尾），大概是因为CoreVideo正在修改底层CVImageBuffer存储，而MTLTexture只是指着那里。
有没有什么办法让CoreVideo在触发该缓冲区之前使用另一个，直到我释放MTLTexture对象？
我目前的解决方法是使用MTLBlitCommandEncoder纹理，但因为我只需要坚持纹理约30毫秒，这似乎是不必要的。
我最近遇到了这个完全相同的问题。 问题是MTLTexture是无效的，除非它拥有CVMetalTextureRef还活着。 在使用MTLTexture的整个过程中，一直保持对CVMetalTextureRef的引用（直到当前渲染周期结束）。
我遇到了同样的问题，但有额外的引用CVMetalTexture对象没有解决这个问题在我的情况。
据我所知，只有当我的金属代码完成处理前一帧之前，我从相机收到一个新的框架才会发生。
看起来，CVMetalTextureCacheCreateTextureFromImage只是在像素缓冲区之上创build一个纹理，这个像素缓冲区是摄像机向其中提供数据的。 因此，从Metal代码asynchronous访问它会导致一些问题。
我决定创build一个MTLTexture（它也是asynchronous的，但速度足够快）。
这里是CVMetalTextureCacheCreateTextureFromImage（）
“该function根据指定的内容创build或返回caching的CoreVideo Metal纹理缓冲区，并将其映射到图像缓冲区，从而在基于设备的图像缓冲区和MTLTexture对象之间创build一个活动的绑定。”，
似乎你的问题取决于你如何pipe理会议获取原始相机数据。
我认为你可以深入地实时分析摄像机会话，以了解你的会话的当前状态（ MetalCameraSession ）：
import AVFoundation import Metal public protocol MetalCameraSessionDelegate { func metalCameraSession(_ session: MetalCameraSession, didReceiveFrameAsTextures: [MTLTexture], withTimestamp: Double) func metalCameraSession(_ session: MetalCameraSession, didUpdateState: MetalCameraSessionState, error: MetalCameraSessionError?) } public final class MetalCameraSession: NSObject { public var frameOrientation: AVCaptureVideoOrientation? { didSet { guard let frameOrientation = frameOrientation, let outputData = outputData, outputData.connection(withMediaType: AVMediaTypeVideo).isVideoOrientationSupported else { return } outputData.connection(withMediaType: AVMediaTypeVideo).videoOrientation = frameOrientation } } public let captureDevicePosition: AVCaptureDevicePosition public var delegate: MetalCameraSessionDelegate? public let pixelFormat: MetalCameraPixelFormat public init(pixelFormat: MetalCameraPixelFormat = .rgb, captureDevicePosition: AVCaptureDevicePosition = .back, delegate: MetalCameraSessionDelegate? = nil) { self.pixelFormat = pixelFormat self.captureDevicePosition = captureDevicePosition self.delegate = delegate super.init(); NotificationCenter.default.addObserver(self, selector: #selector(captureSessionRuntimeError), name: NSNotification.Name.AVCaptureSessionRuntimeError, object: nil) } public func start() { requestCameraAccess() captureSessionQueue.async(execute: { do { self.captureSession.beginConfiguration() try self.initializeInputDevice() try self.initializeOutputData() self.captureSession.commitConfiguration() try self.initializeTextureCache() self.captureSession.startRunning() self.state = .streaming } catch let error as MetalCameraSessionError { self.handleError(error) } catch { print(error.localizedDescription) } }) } public func stop() { captureSessionQueue.async(execute: { self.captureSession.stopRunning() self.state = .stopped }) } fileprivate var state: MetalCameraSessionState = .waiting { didSet { guard state != .error else { return } delegate?.metalCameraSession(self, didUpdateState: state, error: nil) } } fileprivate var captureSession = AVCaptureSession() internal var captureDevice = MetalCameraCaptureDevice() fileprivate var captureSessionQueue = DispatchQueue(label: "MetalCameraSessionQueue", attributes: []) #if arch(i386) || arch(x86_64) #else /// Texture cache we will use for converting frame images to textures internal var textureCache: CVMetalTextureCache? #endif fileprivate var metalDevice = MTLCreateSystemDefaultDevice() internal var inputDevice: AVCaptureDeviceInput? { didSet { if let oldValue = oldValue { captureSession.removeInput(oldValue) } captureSession.addInput(inputDevice) } } internal var outputData: AVCaptureVideoDataOutput? { didSet { if let oldValue = oldValue { captureSession.removeOutput(oldValue) } captureSession.addOutput(outputData) } } fileprivate func requestCameraAccess() { captureDevice.requestAccessForMediaType(AVMediaTypeVideo) { (granted: Bool) -> Void in guard granted else { self.handleError(.noHardwareAccess) return } if self.state != .streaming && self.state != .error { self.state = .ready } } } fileprivate func handleError(_ error: MetalCameraSessionError) { if error.isStreamingError() { state = .error } delegate?.metalCameraSession(self, didUpdateState: state, error: error) } fileprivate func initializeTextureCache() throws { #if arch(i386) || arch(x86_64) throw MetalCameraSessionError.failedToCreateTextureCache #else guard let metalDevice = metalDevice, CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, metalDevice, nil, &textureCache) == kCVReturnSuccess else { throw MetalCameraSessionError.failedToCreateTextureCache } #endif } fileprivate func initializeInputDevice() throws { var captureInput: AVCaptureDeviceInput! guard let inputDevice = captureDevice.device(mediaType: AVMediaTypeVideo, position: captureDevicePosition) else { throw MetalCameraSessionError.requestedHardwareNotFound } do { captureInput = try AVCaptureDeviceInput(device: inputDevice) } catch { throw MetalCameraSessionError.inputDeviceNotAvailable } guard captureSession.canAddInput(captureInput) else { throw MetalCameraSessionError.failedToAddCaptureInputDevice } self.inputDevice = captureInput } fileprivate func initializeOutputData() throws { let outputData = AVCaptureVideoDataOutput() outputData.videoSettings = [ kCVPixelBufferPixelFormatTypeKey as AnyHashable : Int(pixelFormat.coreVideoType) ] outputData.alwaysDiscardsLateVideoFrames = true outputData.setSampleBufferDelegate(self, queue: captureSessionQueue) guard captureSession.canAddOutput(outputData) else { throw MetalCameraSessionError.failedToAddCaptureOutput } self.outputData = outputData } @objc fileprivate func captureSessionRuntimeError() { if state == .streaming { handleError(.captureSessionRuntimeError) } } deinit { NotificationCenter.default.removeObserver(self) } } extension MetalCameraSession: AVCaptureVideoDataOutputSampleBufferDelegate { #if arch(i386) || arch(x86_64) #else private func texture(sampleBuffer: CMSampleBuffer?, textureCache: CVMetalTextureCache?, planeIndex: Int = 0, pixelFormat: MTLPixelFormat = .bgra8Unorm) throws -> MTLTexture { guard let sampleBuffer = sampleBuffer else { throw MetalCameraSessionError.missingSampleBuffer } guard let textureCache = textureCache else { throw MetalCameraSessionError.failedToCreateTextureCache } guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { throw MetalCameraSessionError.failedToGetImageBuffer } let isPlanar = CVPixelBufferIsPlanar(imageBuffer) let width = isPlanar ? CVPixelBufferGetWidthOfPlane(imageBuffer, planeIndex) : CVPixelBufferGetWidth(imageBuffer) let height = isPlanar ? CVPixelBufferGetHeightOfPlane(imageBuffer, planeIndex) : CVPixelBufferGetHeight(imageBuffer) var imageTexture: CVMetalTexture? let result = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, imageBuffer, nil, pixelFormat, width, height, planeIndex, &imageTexture) guard let unwrappedImageTexture = imageTexture, let texture = CVMetalTextureGetTexture(unwrappedImageTexture), result == kCVReturnSuccess else { throw MetalCameraSessionError.failedToCreateTextureFromImage } return texture } private func timestamp(sampleBuffer: CMSampleBuffer?) throws -> Double { guard let sampleBuffer = sampleBuffer else { throw MetalCameraSessionError.missingSampleBuffer } let time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) guard time != kCMTimeInvalid else { throw MetalCameraSessionError.failedToRetrieveTimestamp } return (Double)(time.value) / (Double)(time.timescale); } @objc public func captureOutput(_ captureOutput: AVCaptureOutput!, didOutputSampleBuffer sampleBuffer: CMSampleBuffer!, from connection: AVCaptureConnection!) { do { var textures: [MTLTexture]! switch pixelFormat { case .rgb: let textureRGB = try texture(sampleBuffer: sampleBuffer, textureCache: textureCache) textures = [textureRGB] case .yCbCr: let textureY = try texture(sampleBuffer: sampleBuffer, textureCache: textureCache, planeIndex: 0, pixelFormat: .r8Unorm) let textureCbCr = try texture(sampleBuffer: sampleBuffer, textureCache: textureCache, planeIndex: 1, pixelFormat: .rg8Unorm) textures = [textureY, textureCbCr] } let timestamp = try self.timestamp(sampleBuffer: sampleBuffer) delegate?.metalCameraSession(self, didReceiveFrameAsTextures: textures, withTimestamp: timestamp) } catch let error as MetalCameraSessionError { self.handleError(error) } catch { print(error.localizedDescription) } } #endif }
通过这个类来了解不同的会话types和发生的错误（ MetalCameraSessionTypes ）：
import AVFoundation public enum MetalCameraSessionState { case ready case streaming case stopped case waiting case error } public enum MetalCameraPixelFormat { case rgb case yCbCr var coreVideoType: OSType { switch self { case .rgb: return kCVPixelFormatType_32BGRA case .yCbCr: return kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange } } } public enum MetalCameraSessionError: Error { case noHardwareAccess case failedToAddCaptureInputDevice case failedToAddCaptureOutput case requestedHardwareNotFound case inputDeviceNotAvailable case captureSessionRuntimeError case failedToCreateTextureCache case missingSampleBuffer case failedToGetImageBuffer case failedToCreateTextureFromImage case failedToRetrieveTimestamp public func isStreamingError() -> Bool { switch self { case .noHardwareAccess, .failedToAddCaptureInputDevice, .failedToAddCaptureOutput, .requestedHardwareNotFound, .inputDeviceNotAvailable, .captureSessionRuntimeError: return true default: return false } } public var localizedDescription: String { switch self { case .noHardwareAccess: return "Failed to get access to the hardware for a given media type." case .failedToAddCaptureInputDevice: return "Failed to add a capture input device to the capture session." case .failedToAddCaptureOutput: return "Failed to add a capture output data channel to the capture session." case .requestedHardwareNotFound: return "Specified hardware is not available on this device." case .inputDeviceNotAvailable: return "Capture input device cannot be opened, probably because it is no longer available or because it is in use." case .captureSessionRuntimeError: return "AVCaptureSession runtime error." case .failedToCreateTextureCache: return "Failed to initialize texture cache." case .missingSampleBuffer: return "No sample buffer to convert the image from." case .failedToGetImageBuffer: return "Failed to retrieve an image buffer from camera's output sample buffer." case .failedToCreateTextureFromImage: return "Failed to convert the frame to a Metal texture." case .failedToRetrieveTimestamp: return "Failed to retrieve timestamp from the sample buffer." } } }
然后，您可以使用AVFoundation的AVCaptureDevice的包装器，它具有实例方法而不是类方法（MetalCameraCaptureDevice）：
import AVFoundation internal class MetalCameraCaptureDevice { internal func device(mediaType: String, position: AVCaptureDevicePosition) -> AVCaptureDevice? { guard let devices = AVCaptureDevice.devices(withMediaType: mediaType) as? [AVCaptureDevice] else { return nil } if let index = devices.index(where: { $0.position == position }) { return devices[index] } return nil } internal func requestAccessForMediaType(_ mediaType: String!, completionHandler handler: ((Bool) -> Void)!) { AVCaptureDevice.requestAccess(forMediaType: mediaType, completionHandler: handler) } }
然后你可以有一个自定义viewController类来控制像这样的摄像头（ CameraViewController ）：
import UIKit import Metal internal final class CameraViewController: MTKViewController { var session: MetalCameraSession? override func viewDidLoad() { super.viewDidLoad() session = MetalCameraSession(delegate: self) } override func viewWillAppear(_ animated: Bool) { super.viewWillAppear(animated) session?.start() } override func viewDidDisappear(_ animated: Bool) { super.viewDidDisappear(animated) session?.stop() } } // MARK: - MetalCameraSessionDelegate extension CameraViewController: MetalCameraSessionDelegate { func metalCameraSession(_ session: MetalCameraSession, didReceiveFrameAsTextures textures: [MTLTexture], withTimestamp timestamp: Double) { self.texture = textures[0] } func metalCameraSession(_ cameraSession: MetalCameraSession, didUpdateState state: MetalCameraSessionState, error: MetalCameraSessionError?) { if error == .captureSessionRuntimeError { print(error?.localizedDescription ?? "None") cameraSession.start() } DispatchQueue.main.async { self.title = "Metal camera: \(state)" } print("Session changed state to \(state) with error: \(error?.localizedDescription ?? "None").") } }
最后你的类可以像这个（MTKViewController），你有
public func draw(in: MTKView)
那么你就可以从缓冲区相机中得到你所期望的MTLTexture ：
import UIKit import Metal #if arch(i386) || arch(x86_64) #else import MetalKit #endif open class MTKViewController: UIViewController { open var texture: MTLTexture? open func willRenderTexture(_ texture: inout MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) { } open func didRenderTexture(_ texture: MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) { } override open func loadView() { super.loadView() #if arch(i386) || arch(x86_64) NSLog("Failed creating a default system Metal device, since Metal is not available on iOS Simulator.") #else assert(device != nil, "Failed creating a default system Metal device. Please, make sure Metal is available on your hardware.") #endif initializeMetalView() initializeRenderPipelineState() } fileprivate func initializeMetalView() { #if arch(i386) || arch(x86_64) #else metalView = MTKView(frame: view.bounds, device: device) metalView.delegate = self metalView.framebufferOnly = true metalView.colorPixelFormat = .bgra8Unorm metalView.contentScaleFactor = UIScreen.main.scale metalView.autoresizingMask = [.flexibleWidth, .flexibleHeight] view.insertSubview(metalView, at: 0) #endif } #if arch(i386) || arch(x86_64) #else internal var metalView: MTKView! #endif internal var device = MTLCreateSystemDefaultDevice() internal var renderPipelineState: MTLRenderPipelineState? fileprivate let semaphore = DispatchSemaphore(value: 1) fileprivate func initializeRenderPipelineState() { guard let device = device, let library = device.newDefaultLibrary() else { return } let pipelineDescriptor = MTLRenderPipelineDescriptor() pipelineDescriptor.sampleCount = 1 pipelineDescriptor.colorAttachments[0].pixelFormat = .bgra8Unorm pipelineDescriptor.depthAttachmentPixelFormat = .invalid pipelineDescriptor.vertexFunction = library.makeFunction(name: "mapTexture") pipelineDescriptor.fragmentFunction = library.makeFunction(name: "displayTexture") do { try renderPipelineState = device.makeRenderPipelineState(descriptor: pipelineDescriptor) } catch { assertionFailure("Failed creating a render state pipeline. Can't render the texture without one.") return } } } #if arch(i386) || arch(x86_64) #else extension MTKViewController: MTKViewDelegate { public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) { NSLog("MTKView drawable size will change to \(size)") } public func draw(in: MTKView) { _ = semaphore.wait(timeout: DispatchTime.distantFuture) autoreleasepool { guard var texture = texture, let device = device else { _ = semaphore.signal() return } let commandBuffer = device.makeCommandQueue().makeCommandBuffer() willRenderTexture(&texture, withCommandBuffer: commandBuffer, device: device) render(texture: texture, withCommandBuffer: commandBuffer, device: device) } } private func render(texture: MTLTexture, withCommandBuffer commandBuffer: MTLCommandBuffer, device: MTLDevice) { guard let currentRenderPassDescriptor = metalView.currentRenderPassDescriptor, let currentDrawable = metalView.currentDrawable, let renderPipelineState = renderPipelineState else { semaphore.signal() return } let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: currentRenderPassDescriptor) encoder.pushDebugGroup("RenderFrame") encoder.setRenderPipelineState(renderPipelineState) encoder.setFragmentTexture(texture, at: 0) encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4, instanceCount: 1) encoder.popDebugGroup() encoder.endEncoding() commandBuffer.addScheduledHandler { [weak self] (buffer) in guard let unwrappedSelf = self else { return } unwrappedSelf.didRenderTexture(texture, withCommandBuffer: buffer, device: device) unwrappedSelf.semaphore.signal() } commandBuffer.present(currentDrawable) commandBuffer.commit() } } #endif
现在你有所有的来源，但你也可以find所有的navoshta（作者） GitHUB项目在这里完整的代码和关于这个项目的一个很好的教程， 这里尤其是第二部分，你可以获得纹理的所有评论和说明（你可以在MetalCameraSession类的下面find这个代码）：
guard let unwrappedImageTexture = imageTexture, let texture = CVMetalTextureGetTexture(unwrappedImageTexture), result == kCVReturnSuccess else { throw MetalCameraSessionError.failedToCreateTextureFromImage }
问题可能是由于相机input引起的。如果您的素材与预期输出的帧速率不完全相同，帧速率不匹配将导致奇怪的重影。请尝试禁用自动调整帧速率。
这个问题的其他原因可能是由于以下原因：
关键速度：有一定的速度，与帧速率同步，使他们造成口吃。 帧率越低，问题就越明显。
子像素插值：还有其他情况下，帧之间的子像素插值会导致细节区域在帧之间闪烁。
成功渲染的解决scheme是为您的帧频使用正确的速度（像素/秒），添加足够的运动模糊来隐藏问题，或者减less图像中的细节数量。