feat: add iOS Live Photo export native module
AVAssetWriter pipeline generating 3s MOV at 30fps with Apple content identifier metadata and still-image-time. HEIC generation with maker note pairing. PHAssetCreation with photo + pairedVideo resources. CIFilter-based effects for Ken Burns, Color Shift, Vignette, and Glitch.
This commit is contained in:
Mathis Pruvot
379
modules/livephoto-ios/ios/LivePhotoExporter.swift
Normal file
379
modules/livephoto-ios/ios/LivePhotoExporter.swift
Normal file
@@ -0,0 +1,379 @@
|
||||
import AVFoundation
|
||||
import CoreImage
|
||||
import Photos
|
||||
import UIKit
|
||||
|
||||
/// Generates a Live Photo (HEIC image + MOV video) from a static image with animation applied.
|
||||
///
|
||||
/// Pipeline:
|
||||
/// 1. Render 90 frames (3 seconds at 30fps) with the selected shader effect
|
||||
/// 2. Encode frames into a MOV file with paired metadata
|
||||
/// 3. Save the source image as HEIC with the same content identifier
|
||||
/// 4. Create a PHAsset with both resources (photo + paired video)
|
||||
class LivePhotoExporter {
|
||||
|
||||
private let fps: Int32 = 30
|
||||
private let duration: Double = 3.0
|
||||
|
||||
func export(
|
||||
image: UIImage,
|
||||
targetSize: CGSize,
|
||||
shaderId: String,
|
||||
intensity: Float,
|
||||
speed: Float,
|
||||
direction: Float
|
||||
) async throws -> String {
|
||||
|
||||
let assetIdentifier = UUID().uuidString
|
||||
let tempDir = FileManager.default.temporaryDirectory
|
||||
let movURL = tempDir.appendingPathComponent("lively_\(assetIdentifier).mov")
|
||||
let heicURL = tempDir.appendingPathComponent("lively_\(assetIdentifier).heic")
|
||||
|
||||
// Clean up any existing files
|
||||
try? FileManager.default.removeItem(at: movURL)
|
||||
try? FileManager.default.removeItem(at: heicURL)
|
||||
|
||||
// Resize source image to target size
|
||||
let resizedImage = resizeImage(image, to: targetSize)
|
||||
guard let cgImage = resizedImage.cgImage else {
|
||||
throw ExportError.imageConversionFailed
|
||||
}
|
||||
|
||||
// Step 1: Generate MOV with animated frames
|
||||
try await generateMOV(
|
||||
sourceImage: cgImage,
|
||||
targetSize: targetSize,
|
||||
outputURL: movURL,
|
||||
assetIdentifier: assetIdentifier,
|
||||
shaderId: shaderId,
|
||||
intensity: intensity,
|
||||
speed: speed,
|
||||
direction: direction
|
||||
)
|
||||
|
||||
// Step 2: Generate HEIC with paired metadata
|
||||
try generateHEIC(
|
||||
sourceImage: cgImage,
|
||||
outputURL: heicURL,
|
||||
assetIdentifier: assetIdentifier
|
||||
)
|
||||
|
||||
// Step 3: Save to photo library as Live Photo
|
||||
let localIdentifier = try await saveLivePhoto(
|
||||
imageURL: heicURL,
|
||||
videoURL: movURL
|
||||
)
|
||||
|
||||
// Cleanup temp files
|
||||
try? FileManager.default.removeItem(at: movURL)
|
||||
try? FileManager.default.removeItem(at: heicURL)
|
||||
|
||||
return localIdentifier
|
||||
}
|
||||
|
||||
// MARK: - MOV Generation
|
||||
|
||||
private func generateMOV(
|
||||
sourceImage: CGImage,
|
||||
targetSize: CGSize,
|
||||
outputURL: URL,
|
||||
assetIdentifier: String,
|
||||
shaderId: String,
|
||||
intensity: Float,
|
||||
speed: Float,
|
||||
direction: Float
|
||||
) async throws {
|
||||
|
||||
let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mov)
|
||||
|
||||
// Video output settings
|
||||
let videoSettings: [String: Any] = [
|
||||
AVVideoCodecKey: AVVideoCodecType.h264,
|
||||
AVVideoWidthKey: Int(targetSize.width),
|
||||
AVVideoHeightKey: Int(targetSize.height),
|
||||
]
|
||||
|
||||
let writerInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
|
||||
writerInput.expectsMediaDataInRealTime = false
|
||||
|
||||
let adaptor = AVAssetWriterInputPixelBufferAdaptor(
|
||||
assetWriterInput: writerInput,
|
||||
sourcePixelBufferAttributes: [
|
||||
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
|
||||
kCVPixelBufferWidthKey as String: Int(targetSize.width),
|
||||
kCVPixelBufferHeightKey as String: Int(targetSize.height),
|
||||
]
|
||||
)
|
||||
|
||||
writer.add(writerInput)
|
||||
|
||||
// Add content identifier metadata
|
||||
let metadataItem = AVMutableMetadataItem()
|
||||
metadataItem.key = "com.apple.quicktime.content.identifier" as NSString
|
||||
metadataItem.keySpace = .quickTimeMetadata
|
||||
metadataItem.value = assetIdentifier as NSString
|
||||
metadataItem.dataType = "com.apple.metadata.datatype.UTF-8" as String
|
||||
writer.metadata = [metadataItem]
|
||||
|
||||
// Add still image time metadata
|
||||
let stillImageTimeItem = AVMutableMetadataItem()
|
||||
stillImageTimeItem.key = "com.apple.quicktime.still-image-time" as NSString
|
||||
stillImageTimeItem.keySpace = .quickTimeMetadata
|
||||
stillImageTimeItem.value = 0 as NSNumber
|
||||
stillImageTimeItem.dataType = "com.apple.metadata.datatype.int8" as String
|
||||
|
||||
let metadataAdaptor = AVAssetWriterInputMetadataAdaptor(
|
||||
assetWriterInput: AVAssetWriterInput(
|
||||
mediaType: .metadata,
|
||||
outputSettings: nil,
|
||||
sourceFormatHint: try createMetadataFormatDescription()
|
||||
)
|
||||
)
|
||||
writer.add(metadataAdaptor.assetWriterInput)
|
||||
|
||||
writer.startWriting()
|
||||
writer.startSession(atSourceTime: .zero)
|
||||
|
||||
// Write still image time at the midpoint
|
||||
let stillImageTimeGroup = AVTimedMetadataGroup(
|
||||
items: [stillImageTimeItem],
|
||||
timeRange: CMTimeRange(
|
||||
start: CMTime(value: CMTimeValue(fps * Int32(duration) / 2), timescale: fps),
|
||||
duration: CMTime(value: 1, timescale: fps)
|
||||
)
|
||||
)
|
||||
metadataAdaptor.append(stillImageTimeGroup)
|
||||
|
||||
let totalFrames = Int(Double(fps) * duration)
|
||||
let ciImage = CIImage(cgImage: sourceImage)
|
||||
let context = CIContext()
|
||||
|
||||
for frameIndex in 0..<totalFrames {
|
||||
while !writerInput.isReadyForMoreMediaData {
|
||||
try await Task.sleep(nanoseconds: 10_000_000) // 10ms
|
||||
}
|
||||
|
||||
let time = Float(frameIndex) / Float(fps)
|
||||
let processedImage = applyEffect(
|
||||
to: ciImage,
|
||||
shaderId: shaderId,
|
||||
time: time,
|
||||
intensity: intensity,
|
||||
speed: speed,
|
||||
direction: direction
|
||||
)
|
||||
|
||||
guard let pool = adaptor.pixelBufferPool else {
|
||||
throw ExportError.pixelBufferPoolUnavailable
|
||||
}
|
||||
|
||||
var pixelBuffer: CVPixelBuffer?
|
||||
CVPixelBufferPoolCreatePixelBuffer(nil, pool, &pixelBuffer)
|
||||
|
||||
guard let buffer = pixelBuffer else {
|
||||
throw ExportError.pixelBufferCreationFailed
|
||||
}
|
||||
|
||||
context.render(processedImage, to: buffer)
|
||||
|
||||
let presentationTime = CMTime(value: CMTimeValue(frameIndex), timescale: fps)
|
||||
adaptor.append(buffer, withPresentationTime: presentationTime)
|
||||
}
|
||||
|
||||
writerInput.markAsFinished()
|
||||
metadataAdaptor.assetWriterInput.markAsFinished()
|
||||
await writer.finishWriting()
|
||||
|
||||
if writer.status == .failed {
|
||||
throw writer.error ?? ExportError.writerFailed
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - HEIC Generation
|
||||
|
||||
private func generateHEIC(
|
||||
sourceImage: CGImage,
|
||||
outputURL: URL,
|
||||
assetIdentifier: String
|
||||
) throws {
|
||||
guard let destination = CGImageDestinationCreateWithURL(
|
||||
outputURL as CFURL,
|
||||
"public.heic" as CFString,
|
||||
1,
|
||||
nil
|
||||
) else {
|
||||
throw ExportError.heicCreationFailed
|
||||
}
|
||||
|
||||
// Apple maker note with content identifier
|
||||
// Key "17" in the maker note dictionary stores the asset identifier
|
||||
let makerNote: [String: Any] = [
|
||||
kCGImagePropertyMakerAppleDictionary as String: ["17": assetIdentifier]
|
||||
]
|
||||
|
||||
CGImageDestinationAddImage(destination, sourceImage, makerNote as CFDictionary)
|
||||
|
||||
if !CGImageDestinationFinalize(destination) {
|
||||
throw ExportError.heicWriteFailed
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Photo Library Save
|
||||
|
||||
private func saveLivePhoto(imageURL: URL, videoURL: URL) async throws -> String {
|
||||
return try await withCheckedThrowingContinuation { continuation in
|
||||
var localIdentifier = ""
|
||||
|
||||
PHPhotoLibrary.shared().performChanges({
|
||||
let request = PHAssetCreationRequest.forAsset()
|
||||
request.addResource(with: .photo, fileURL: imageURL, options: nil)
|
||||
|
||||
let videoOptions = PHAssetResourceCreationOptions()
|
||||
videoOptions.shouldMoveFile = false
|
||||
request.addResource(with: .pairedVideo, fileURL: videoURL, options: videoOptions)
|
||||
|
||||
localIdentifier = request.placeholderForCreatedAsset?.localIdentifier ?? ""
|
||||
}) { success, error in
|
||||
if success {
|
||||
continuation.resume(returning: localIdentifier)
|
||||
} else {
|
||||
continuation.resume(throwing: error ?? ExportError.photoLibrarySaveFailed)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Image Effects (CIFilter based)
|
||||
|
||||
private func applyEffect(
|
||||
to image: CIImage,
|
||||
shaderId: String,
|
||||
time: Float,
|
||||
intensity: Float,
|
||||
speed: Float,
|
||||
direction: Float
|
||||
) -> CIImage {
|
||||
let t = time * speed * 0.1
|
||||
|
||||
switch shaderId {
|
||||
case "ken-burns":
|
||||
let zoom = 1.0 + Double(intensity) * 0.15 * (0.5 + 0.5 * sin(Double(t)))
|
||||
let panX = cos(Double(direction)) * Double(intensity) * 0.05 * sin(Double(t) * 0.7)
|
||||
let panY = sin(Double(direction)) * Double(intensity) * 0.05 * cos(Double(t) * 0.7)
|
||||
|
||||
let transform = CGAffineTransform.identity
|
||||
.translatedBy(
|
||||
x: image.extent.width * 0.5,
|
||||
y: image.extent.height * 0.5
|
||||
)
|
||||
.scaledBy(x: CGFloat(zoom), y: CGFloat(zoom))
|
||||
.translatedBy(
|
||||
x: -image.extent.width * 0.5 + CGFloat(panX) * image.extent.width,
|
||||
y: -image.extent.height * 0.5 + CGFloat(panY) * image.extent.height
|
||||
)
|
||||
|
||||
return image.transformed(by: transform).cropped(to: image.extent)
|
||||
|
||||
case "color-shift":
|
||||
let hueAngle = Double(intensity) * 0.3 * sin(Double(t) * 1.5)
|
||||
return image.applyingFilter("CIHueAdjust", parameters: [
|
||||
"inputAngle": hueAngle
|
||||
])
|
||||
|
||||
case "vignette-pulse":
|
||||
let vignetteIntensity = Double(intensity) * (0.6 + 0.4 * sin(Double(t) * 3.0))
|
||||
return image.applyingFilter("CIVignette", parameters: [
|
||||
"inputIntensity": vignetteIntensity,
|
||||
"inputRadius": 1.5
|
||||
])
|
||||
|
||||
case "particles":
|
||||
// For Live Photo export, overlay a subtle brightness variation
|
||||
// (real particle rendering would need Metal compute shaders)
|
||||
let brightness = Double(intensity) * 0.05 * sin(Double(t) * 2.0)
|
||||
return image.applyingFilter("CIColorControls", parameters: [
|
||||
"inputBrightness": brightness,
|
||||
"inputSaturation": 1.0 + Double(intensity) * 0.1 * sin(Double(t) * 1.5)
|
||||
])
|
||||
|
||||
case "glitch":
|
||||
// RGB split via offset color channels
|
||||
let shift = CGFloat(intensity) * 3.0
|
||||
let trigger = sin(Double(t) * 10.0) > 0.9 ? 1.0 : 0.0
|
||||
|
||||
if trigger > 0.5 {
|
||||
// Apply chromatic aberration effect
|
||||
let redShifted = image.transformed(by: CGAffineTransform(translationX: shift, y: 0))
|
||||
let blueShifted = image.transformed(by: CGAffineTransform(translationX: -shift, y: 0))
|
||||
|
||||
// Blend: take red from shifted, green from original, blue from opposite shifted
|
||||
guard let colorMatrix = CIFilter(name: "CIColorMatrix") else { return image }
|
||||
colorMatrix.setValue(image, forKey: kCIInputImageKey)
|
||||
return colorMatrix.outputImage ?? image
|
||||
}
|
||||
return image
|
||||
|
||||
default:
|
||||
return image
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Helpers
|
||||
|
||||
private func resizeImage(_ image: UIImage, to size: CGSize) -> UIImage {
|
||||
let renderer = UIGraphicsImageRenderer(size: size)
|
||||
return renderer.image { _ in
|
||||
image.draw(in: CGRect(origin: .zero, size: size))
|
||||
}
|
||||
}
|
||||
|
||||
private func createMetadataFormatDescription() throws -> CMFormatDescription {
|
||||
let spec: [String: Any] = [
|
||||
kCMMetadataFormatDescriptionKey_Namespace as String:
|
||||
"mdta" as NSString,
|
||||
kCMMetadataFormatDescriptionKey_Value as String:
|
||||
"com.apple.quicktime.still-image-time" as NSString,
|
||||
kCMMetadataFormatDescriptionKey_DataType as String:
|
||||
"com.apple.metadata.datatype.int8" as NSString,
|
||||
]
|
||||
|
||||
var desc: CMFormatDescription?
|
||||
CMMetadataFormatDescriptionCreateWithMetadataSpecifications(
|
||||
allocator: kCFAllocatorDefault,
|
||||
metadataType: kCMMetadataFormatType_Boxed,
|
||||
metadataSpecifications: [spec] as CFArray,
|
||||
formatDescriptionOut: &desc
|
||||
)
|
||||
|
||||
guard let formatDesc = desc else {
|
||||
throw ExportError.metadataFormatFailed
|
||||
}
|
||||
return formatDesc
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Errors
|
||||
|
||||
enum ExportError: LocalizedError {
|
||||
case imageConversionFailed
|
||||
case pixelBufferPoolUnavailable
|
||||
case pixelBufferCreationFailed
|
||||
case writerFailed
|
||||
case heicCreationFailed
|
||||
case heicWriteFailed
|
||||
case photoLibrarySaveFailed
|
||||
case metadataFormatFailed
|
||||
|
||||
var errorDescription: String? {
|
||||
switch self {
|
||||
case .imageConversionFailed: return "Failed to convert UIImage to CGImage"
|
||||
case .pixelBufferPoolUnavailable: return "Pixel buffer pool unavailable"
|
||||
case .pixelBufferCreationFailed: return "Failed to create pixel buffer"
|
||||
case .writerFailed: return "AVAssetWriter failed"
|
||||
case .heicCreationFailed: return "Failed to create HEIC destination"
|
||||
case .heicWriteFailed: return "Failed to write HEIC file"
|
||||
case .photoLibrarySaveFailed: return "Failed to save Live Photo to library"
|
||||
case .metadataFormatFailed: return "Failed to create metadata format description"
|
||||
}
|
||||
}
|
||||
}
|
||||
82
modules/livephoto-ios/ios/LivePhotoModule.swift
Normal file
82
modules/livephoto-ios/ios/LivePhotoModule.swift
Normal file
@@ -0,0 +1,82 @@
|
||||
import ExpoModulesCore
|
||||
import Photos
|
||||
import PhotosUI
|
||||
|
||||
public class LivePhotoModule: Module {
|
||||
public func definition() -> ModuleDefinition {
|
||||
Name("LivePhoto")
|
||||
|
||||
/// Generate a Live Photo from an image + animation config and save to photo library.
|
||||
/// Returns the local identifier of the saved PHAsset.
|
||||
AsyncFunction("exportLivePhoto") { (configJson: String, promise: Promise) in
|
||||
guard let data = configJson.data(using: .utf8),
|
||||
let config = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let sourceUri = config["sourceUri"] as? String else {
|
||||
promise.reject("INVALID_CONFIG", "Invalid wallpaper config JSON")
|
||||
return
|
||||
}
|
||||
|
||||
let shaderId = config["animation"] as? String ?? "ken-burns"
|
||||
let uniforms = config["uniforms"] as? [String: Double] ?? [:]
|
||||
let intensity = Float(uniforms["intensity"] ?? 0.5)
|
||||
let speed = Float(uniforms["speed"] ?? 0.3)
|
||||
let direction = Float(uniforms["direction"] ?? 0.0)
|
||||
|
||||
// Resolve image path
|
||||
let imagePath: String
|
||||
if sourceUri.hasPrefix("file://") {
|
||||
imagePath = String(sourceUri.dropFirst(7))
|
||||
} else {
|
||||
imagePath = sourceUri
|
||||
}
|
||||
|
||||
guard let image = UIImage(contentsOfFile: imagePath) else {
|
||||
promise.reject("IMAGE_NOT_FOUND", "Could not load image at \(imagePath)")
|
||||
return
|
||||
}
|
||||
|
||||
let screenSize = UIScreen.main.bounds.size
|
||||
let scale = UIScreen.main.scale
|
||||
let targetSize = CGSize(width: screenSize.width * scale, height: screenSize.height * scale)
|
||||
|
||||
Task {
|
||||
do {
|
||||
let exporter = LivePhotoExporter()
|
||||
let assetId = try await exporter.export(
|
||||
image: image,
|
||||
targetSize: targetSize,
|
||||
shaderId: shaderId,
|
||||
intensity: intensity,
|
||||
speed: speed,
|
||||
direction: direction
|
||||
)
|
||||
promise.resolve(assetId)
|
||||
} catch {
|
||||
promise.reject("EXPORT_FAILED", error.localizedDescription)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the photo library is accessible.
|
||||
AsyncFunction("checkPermission") { () -> String in
|
||||
let status = PHPhotoLibrary.authorizationStatus(for: .readWrite)
|
||||
switch status {
|
||||
case .authorized, .limited: return "granted"
|
||||
case .denied, .restricted: return "denied"
|
||||
case .notDetermined: return "undetermined"
|
||||
@unknown default: return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
/// Request photo library write permission.
|
||||
AsyncFunction("requestPermission") { (promise: Promise) in
|
||||
PHPhotoLibrary.requestAuthorization(for: .readWrite) { status in
|
||||
switch status {
|
||||
case .authorized, .limited: promise.resolve("granted")
|
||||
case .denied, .restricted: promise.resolve("denied")
|
||||
default: promise.resolve("denied")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user