/* * PROJECT: FLARManager * http://transmote.com/flar * Copyright 2009, Eric Socolofsky * -------------------------------------------------------------------------------- * This work complements FLARToolkit, developed by Saqoosha as part of the Libspark project. * http://www.libspark.org/wiki/saqoosha/FLARToolKit * FLARToolkit is Copyright (C)2008 Saqoosha, * and is ported from NYARToolkit, which is ported from ARToolkit. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this framework; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA * * For further information please contact: * * http://transmote.com/flar * */ package com.transmote.flar { import __AS3__.vec.Vector; import com.transmote.flar.marker.FLARMarker; import com.transmote.flar.marker.FLARMarkerEvent; import com.transmote.flar.marker.FLARMarkerOutline; import com.transmote.flar.pattern.FLARPattern; import com.transmote.flar.pattern.FLARPatternLoader; import com.transmote.flar.source.FLARCameraSource; import com.transmote.flar.source.FLARLoaderSource; import com.transmote.flar.source.IFLARSource; import com.transmote.flar.utils.FLARMatrixSmoother_Average; import com.transmote.flar.utils.IFLARMatrixSmoother; import flash.display.Bitmap; import flash.display.DisplayObjectContainer; import flash.display.Sprite; import flash.events.Event; import flash.events.EventDispatcher; import flash.events.IOErrorEvent; import flash.events.SecurityErrorEvent; import flash.geom.Point; import flash.geom.Rectangle; import flash.net.URLLoader; import flash.net.URLLoaderDataFormat; import flash.net.URLRequest; import flash.utils.ByteArray; import org.libspark.flartoolkit.FLARException; import org.libspark.flartoolkit.core.param.FLARParam; import org.libspark.flartoolkit.core.raster.rgb.FLARRgbRaster_BitmapData; import org.libspark.flartoolkit.core.transmat.FLARTransMatResult; import org.libspark.flartoolkit.detector.FLARMultiMarkerDetector; import org.libspark.flartoolkit.detector.FLARMultiMarkerDetectorResult; /** *

* Manager for computer vision applications using FLARToolkit * ( * http://www.libspark.org/wiki/saqoosha/FLARToolKit/en). *

*

* Basic usage is as follows: * Pass a path to a camera parameters file and a list of FLARPatterns to the constructor. * Optionally pass an IFLARSource to use as the source image for marker detection; * FLARManager will by default create a FLARCameraSource that uses the first available camera. * Alternatively, FLARManager can be initialized using an xml file that specifies the above and other settings. *

*

* Assign event listeners to FLARManager for MARKER_ADDED, * MARKER_UPDATED, and MARKER_REMOVED FLARMarkerEvents. * These FLARMarkerEvents encapsulate the FLARMarker instances that they refer to. * Alternatively, it is possible to retrieve all active markers * directly from FLARManager, via FLARManager.activeMarkers. *

*

* FLARMarkers are simple objects that contain information about detected markers * provided by FLARToolkit. FLARManager maintains a list of active markers, * and updates the list and the markers within every frame. *

* * @author Eric Socolofsky * @url http://transmote.com/flar * @see com.transmote.flar.marker.FLARMarkerEvent * @see com.transmote.flar.source.FLARCameraSource * @see com.transmote.flar.utils.FLARProxy */ public class FLARManager extends EventDispatcher { private var _cameraParams:FLARParam; private var _flarSource:IFLARSource; private var _threshold:Number = 80; private var _markerUpdateThreshold:Number = 80; private var _smoothing:int = 3; private var _smoother:IFLARMatrixSmoother; private var _mirrorDisplay:Boolean; private var _thresholdSourceDisplay:Boolean; private var patternLoader:FLARPatternLoader; private var allPatterns:Vector.; private var markerDetector:FLARMultiMarkerDetector; private var flarRaster:FLARRgbRaster_BitmapData; private var thresholdSourceBitmap:Bitmap; private var enterframer:Sprite; private var _activeMarkers:Vector.; private var bInited:Boolean; private var bCameraParamsLoaded:Boolean; private var bPatternsLoaded:Boolean; /** * Constructor. * FLARManager can be initialized either by passing in the camera parameters file path and a list of FLARPattern instances, * or by specifying a config file path to FLARManager.initFromFile. * * @param source IFLARSource instance to use as source image for marker detection. * if null, FLARManager will create a camera capture source. * @param cameraParamsPath camera parameters filename. * @param patterns list of FLARPatterns to detect. */ public function FLARManager (source:IFLARSource=null, cameraParamsPath:String="", patterns:Vector.=null) { if (source) { this._flarSource = source; this._mirrorDisplay = source.mirrored; } else { this._flarSource = new FLARCameraSource(); } if (cameraParamsPath && patterns) { this.init(cameraParamsPath, patterns); } } /** * load FLARManager configuration from an xml file. * this is the preferred method of FLARManager instantiation. * * @param flarConfigPath path to the FLARManager configuration xml file. */ public function initFromFile (flarConfigPath:String) :void { var loader:URLLoader = new URLLoader(); loader.addEventListener(IOErrorEvent.IO_ERROR, this.onFlarConfigLoadError); loader.addEventListener(SecurityErrorEvent.SECURITY_ERROR, this.onFlarConfigLoadError); loader.addEventListener(Event.COMPLETE, this.onFlarConfigLoaded); loader.load(new URLRequest(flarConfigPath)); } //--------------------------------------// /** * Vector of all currently-active markers. */ public function get activeMarkers () :Vector. { return this._activeMarkers; } /** * FLARParam used by this FLARManager. * can be used to instantiate a FLARCamera3D for use with Papervision. */ public function get cameraParams () :FLARParam { return this._cameraParams; } /** * IFLARSource instance this FLARManager is using as the source image for marker detection. */ public function get flarSource () :IFLARSource { return this._flarSource; } /** * pixels in source image with a brightness <= to this.threshold are candidates for marker outline detection. * increase to increase likelihood of marker detection; * increasing too high will cause engine to incorrectly detect non-existent markers. * defaults to 80 (values can range from 0 to 255). */ public function get threshold () :uint { return this._threshold; } public function set threshold (val:uint) :void { this._threshold = val; } /** * if a detected marker is within this distance from an active marker, * FLARManager considers the detected marker to be an update of the active marker. * else, the detected marker is a new marker. */ public function get markerUpdateThreshold () :Number { return this._markerUpdateThreshold; } public function set markerUpdateThreshold (val:Number) :void { this._markerUpdateThreshold = val; } /** * apply a smoothing algorithm to transformation matrices generated by FLARToolkit. * smoothing is equal to the number of frames over which FLARManager * will average transformation matrices; the larger the number, the smoother the animation, * and the slower the response time between marker position/orientation changes. * a value of 0 turns smoothing off. */ public function get smoothing () :int { return this._smoothing; } public function set smoothing (val:int) :void { this._smoothing = val; } /** * IFLARMatrixSmoother to use to apply smoothing to transformation matrices generated by FLARToolkit. */ public function get smoother () :IFLARMatrixSmoother { return this._smoother; } public function set smoother (val:IFLARMatrixSmoother) :void { this._smoother = val; } /** * set to true to flip the camera image horizontally; * this value is passed on to this.flarSource; * note that if an IFLARSource is specified after mirrorDisplay is set, * the 'mirrored' property of the new IFLARSource will overwrite this value. */ public function get mirrorDisplay () :Boolean { return this._mirrorDisplay; } public function set mirrorDisplay (val:Boolean) :void { this._mirrorDisplay = val; if (this.flarSource) { this.flarSource.mirrored = this._mirrorDisplay; } } /** * the number of patterns loaded for detection. */ public function get numLoadedPatterns () :int { return this.patternLoader.loadedPatterns.length; } /** * display the source BitmapData used by FLARToolkit post-thresholding. * useful for debugging. */ public function get thresholdSourceDisplay () :Boolean { return this._thresholdSourceDisplay; } public function set thresholdSourceDisplay (val:Boolean) :void { this._thresholdSourceDisplay = val; if (this._thresholdSourceDisplay) { try { if (!this.thresholdSourceBitmap) { if (!this.markerDetector.thresholdedBitmapData) { throw new Error(); } this.thresholdSourceBitmap = new Bitmap(this.markerDetector.thresholdedBitmapData); Sprite(this._flarSource).addChild(this.thresholdSourceBitmap); } } catch (e:Error) { this.thresholdSourceBitmap = null; return; } } } //-------------------------------------// //---------------------------------// /** * begin detecting markers once per frame. * this method is called automatically on initialization. * @return false if FLARManager is not yet initialized; else true. */ public function activate () :Boolean { if (!this.bInited) { return false; } if (!this.enterframer) { this.enterframer = new Sprite(); } this.enterframer.addEventListener(Event.ENTER_FRAME, this.onEnterFrame, false, 0, true); this._activeMarkers = new Vector.(); return true; } /** * stop detecting markers. */ public function deactivate () :void { if (this.enterframer) { this.enterframer.removeEventListener(Event.ENTER_FRAME, this.onEnterFrame); } this._activeMarkers = null; } //--------------------------------// //---------------------------------// private function onEnterFrame (evt:Event) :void { if (!this.updateSource()) { return; } this.detectMarkers(); } private function updateSource () :Boolean { try { // ensure this.flarRaster has been initialized if (this.flarRaster == null) { this.flarRaster = new FLARRgbRaster_BitmapData(this.flarSource.source); } } catch (e:Error) { // this.flarSource not yet fully initialized return false; } // update source image this.flarSource.update(); return true; } private function detectMarkers () :void { var numFoundMarkers:int = 0; try { // detect marker(s) numFoundMarkers = this.markerDetector.detectMarkerLite(this.flarRaster, this.threshold); } catch (e:FLARException) { // error in FLARToolkit processing; send to console trace(e); return; } //trace("numFoundMarkers:"+numFoundMarkers); var activeMarker:FLARMarker; var i:uint; if (numFoundMarkers == 0) { // if no markers found, remove any existing markers and exit i = this._activeMarkers.length; while (i--) { activeMarker = this._activeMarkers.splice(i, 1)[0]; this.dispatchEvent(new FLARMarkerEvent(FLARMarkerEvent.MARKER_REMOVED, activeMarker)); return; } } // build list of detected markers var detectedMarkers:Vector. = new Vector.(); var detectedMarkerResult:FLARMultiMarkerDetectorResult; var patternIndex:int; var detectedPattern:FLARPattern; var confidence:Number; var deprecatedOutline:FLARMarkerOutline; var transmat:FLARTransMatResult; i = numFoundMarkers; while (i--) { detectedMarkerResult = this.markerDetector.getResult(i); patternIndex = this.markerDetector.getARCodeIndex(i); detectedPattern = this.allPatterns[patternIndex]; confidence = this.markerDetector.getConfidence(i); if (confidence < detectedPattern.minConfidence) { // detected marker's confidence is below the minimum required confidence for its pattern. continue; } transmat = new FLARTransMatResult(); try { this.markerDetector.getTransmationMatrix(i, transmat); } catch (e:Error) { // FLARException happens with rotationX of approx -60 and +60, and rotY&Z of 0... continue; } deprecatedOutline = new FLARMarkerOutline(detectedMarkerResult.square, transmat, this.flarSource.resultsToDisplayRatio); detectedMarkers.push(new FLARMarker(detectedMarkerResult, transmat, this.flarSource, deprecatedOutline)); } // compare detected markers against active markers i = detectedMarkers.length; var j:uint; var detectedMarker:FLARMarker; var closestMarker:FLARMarker; var closestDist:Number = Number.POSITIVE_INFINITY; var dist:Number; var updatedMarkers:Vector. = new Vector.(); var newMarkers:Vector. = new Vector.(); while (i--) { j = this._activeMarkers.length; detectedMarker = detectedMarkers[i]; closestMarker = null; closestDist = Number.POSITIVE_INFINITY; while (j--) { activeMarker = this._activeMarkers[j]; if (detectedMarker.patternId == activeMarker.patternId) { dist = Point.distance(detectedMarker.centerpoint3D, activeMarker.targetCenterpoint3D); if (dist < closestDist && dist < this._markerUpdateThreshold) { closestMarker = activeMarker; closestDist = dist; } } } if (closestMarker) { // updated marker closestMarker.copy(detectedMarker); detectedMarker.dispose(); if (this._smoothing) { if (!this._smoother) { // TODO: log as a WARN-level error trace("no smoother set; specify FLARManager.smoother to enable smoothing."); } else { closestMarker.applySmoothing(this._smoother, this._smoothing); } } updatedMarkers.push(closestMarker); this.dispatchEvent(new FLARMarkerEvent(FLARMarkerEvent.MARKER_UPDATED, closestMarker)); } else { // new marker newMarkers.push(detectedMarker); detectedMarker.setSessionId(); this.dispatchEvent(new FLARMarkerEvent(FLARMarkerEvent.MARKER_ADDED, detectedMarker)); } } i = this._activeMarkers.length; var removedMarker:FLARMarker; while (i--) { activeMarker = this._activeMarkers[i]; if (updatedMarkers.indexOf(activeMarker) == -1) { // removed marker removedMarker = this._activeMarkers.splice(i, 1)[0]; this.dispatchEvent(new FLARMarkerEvent(FLARMarkerEvent.MARKER_REMOVED, removedMarker)); removedMarker.dispose(); } } this._activeMarkers = this._activeMarkers.concat(newMarkers); } //--------------------------------// //---------------------------------// private function onFlarConfigLoadError (evt:Event) :void { var errorText:String = "character list load error."; if (evt is IOErrorEvent) { errorText = IOErrorEvent(evt).text; } else if (evt is SecurityErrorEvent) { errorText = SecurityErrorEvent(evt).text; } this.onFlarConfigLoaded(evt, new Error(errorText)); } private function onFlarConfigLoaded (evt:Event, error:Error=null) :void { var loader:URLLoader = evt.target as URLLoader; loader.removeEventListener(IOErrorEvent.IO_ERROR, this.onFlarConfigLoadError); loader.removeEventListener(SecurityErrorEvent.SECURITY_ERROR, this.onFlarConfigLoadError); loader.removeEventListener(Event.COMPLETE, this.onFlarConfigLoaded); if (error) { throw error; } this.parseFlarConfigFile(new XML(loader.data as String)); this.dispatchEvent(new Event(Event.COMPLETE)); } private function parseFlarConfigFile (configFileXML:XML) :void { // source settings var sourceInitStruct:SourceInitStruct = new SourceInitStruct(); var sourceWidth:int = parseInt(configFileXML.flarSourceSettings.@sourceWidth); if (!isNaN(sourceWidth) && sourceWidth > 0) { sourceInitStruct.sourceWidth = sourceWidth; } var sourceHeight:int = parseInt(configFileXML.flarSourceSettings.@sourceHeight); if (!isNaN(sourceHeight) && sourceHeight > 0) { sourceInitStruct.sourceHeight = sourceHeight; } var displayWidth:int = parseInt(configFileXML.flarSourceSettings.@displayWidth); if (!isNaN(displayWidth) && displayWidth > 0) { sourceInitStruct.displayWidth = displayWidth; } var displayHeight:int = parseInt(configFileXML.flarSourceSettings.@displayHeight); if (!isNaN(displayHeight) && displayHeight > 0) { sourceInitStruct.displayHeight = displayHeight; } var framerate:Number = parseFloat(configFileXML.flarSourceSettings.@framerate); if (!isNaN(framerate) && framerate > 0) { sourceInitStruct.framerate = framerate; } var downsampleRatio:Number = parseFloat(configFileXML.flarSourceSettings.@downsampleRatio); if (!isNaN(downsampleRatio) && downsampleRatio > 0) { sourceInitStruct.downsampleRatio = downsampleRatio; } sourceInitStruct.loaderPath = configFileXML.flarSourceSettings.@loaderPath; sourceInitStruct.useProxy = configFileXML.flarSourceSettings.@useProxy == "true"; // miscellaneous FLARManager settings var mirrorDisplayStr:String = configFileXML.flarManagerSettings.@mirrorDisplay.toString(); this.mirrorDisplay = mirrorDisplayStr && mirrorDisplayStr.toLowerCase() == "true"; var markerUpdateThresholdVal:Number = parseFloat(configFileXML.flarManagerSettings.@markerUpdateThreshold); if (!isNaN(markerUpdateThresholdVal) && markerUpdateThresholdVal > 0) { this.markerUpdateThreshold = markerUpdateThresholdVal; } var smoothingVal:int = parseInt(configFileXML.flarManagerSettings.@smoothing); if (!isNaN(smoothingVal) && smoothingVal > 0) { this.smoothing = smoothingVal; } // camera parameters file var cameraParamsPath:String = configFileXML.cameraParamsFile.@path; // pattern list var resolution:int = parseInt(configFileXML.patterns.@resolution); var patternToBorderRatio:Number = parseFloat(configFileXML.patterns.@patternToBorderRatio); var minConfidence:Number = parseFloat(configFileXML.patterns.@minConfidence); var patterns:Vector. = new Vector.(); var patternPath:String; for each (var pattern:XML in configFileXML.patterns.pattern) { patterns.push(new FLARPattern(pattern.@path, resolution, patternToBorderRatio, NaN, minConfidence)); } this.init(cameraParamsPath, patterns, sourceInitStruct); } private function init (cameraParamsPath:String, patterns:Vector., sourceInitStruct:SourceInitStruct=null) :void { if (!sourceInitStruct) { sourceInitStruct = new SourceInitStruct(); } this.initFlarSource(sourceInitStruct); this.loadCameraParams(cameraParamsPath); this.allPatterns = patterns; this.loadPatterns(this.allPatterns); } private function initFlarSource (sourceInitStruct:SourceInitStruct) :void { var sourceParent:DisplayObjectContainer; var sourceAsSprite:Sprite; var sourceIndex:int; if (this._flarSource) { sourceAsSprite = this._flarSource as Sprite; sourceParent = sourceAsSprite.parent; } if (sourceInitStruct.useProxy) { if (sourceParent) { // if placeholder IFLARSource was already added to the display list, remove it... sourceIndex = sourceParent.getChildIndex(sourceAsSprite); sourceParent.removeChild(sourceAsSprite); } // TODO: redesign FLARProxy as IFLARSource, and init here if (sourceParent) { // ...and replace it with the new FLARLoaderSource. sourceParent.addChildAt(Sprite(this._flarSource), sourceIndex); } } else if (sourceInitStruct.loaderPath) { if (sourceParent) { // if placeholder IFLARSource was already added to the display list, remove it... sourceIndex = sourceParent.getChildIndex(sourceAsSprite); sourceParent.removeChild(sourceAsSprite); } this._flarSource = new FLARLoaderSource( sourceInitStruct.loaderPath, sourceInitStruct.sourceWidth, sourceInitStruct.sourceHeight, sourceInitStruct.downsampleRatio); if (sourceParent) { // ...and replace it with the new FLARLoaderSource. sourceParent.addChildAt(Sprite(this._flarSource), sourceIndex); } } else { FLARCameraSource(this._flarSource).init( sourceInitStruct.sourceWidth, sourceInitStruct.sourceHeight, sourceInitStruct.framerate, this._mirrorDisplay, sourceInitStruct.displayWidth, sourceInitStruct.displayHeight, sourceInitStruct.downsampleRatio); } } private function loadCameraParams (cameraParamsPath:String) :void { var loader:URLLoader = new URLLoader(); loader.dataFormat = URLLoaderDataFormat.BINARY; loader.addEventListener(IOErrorEvent.IO_ERROR, this.onCameraParamsLoadError); loader.addEventListener(SecurityErrorEvent.SECURITY_ERROR, this.onCameraParamsLoadError); loader.addEventListener(Event.COMPLETE, this.onCameraParamsLoaded); loader.load(new URLRequest(cameraParamsPath)); } private function onCameraParamsLoadError (evt:Event) :void { var errorText:String = "Camera params load error."; if (evt is IOErrorEvent) { errorText = IOErrorEvent(evt).text; } else if (evt is SecurityErrorEvent) { errorText = SecurityErrorEvent(evt).text; } this.onCameraParamsLoaded(evt, new Error(errorText)); } private function onCameraParamsLoaded (evt:Event, error:Error=null) :void { var loader:URLLoader = evt.target as URLLoader; loader.removeEventListener(IOErrorEvent.IO_ERROR, this.onCameraParamsLoadError); loader.removeEventListener(SecurityErrorEvent.SECURITY_ERROR, this.onCameraParamsLoadError); loader.removeEventListener(Event.COMPLETE, this.onCameraParamsLoaded); if (error) { throw error; } this._cameraParams = new FLARParam(); this._cameraParams.loadARParam(ByteArray(loader.data)); var sourceSize:Rectangle = this.flarSource.sourceSize; this._cameraParams.changeScreenSize(sourceSize.width, sourceSize.height); this.bCameraParamsLoaded = true; this.checkForInitComplete(); } private function loadPatterns (patterns:Vector.) :void { this.patternLoader = new FLARPatternLoader(); this.patternLoader.addEventListener(Event.INIT, this.onPatternsLoaded); this.patternLoader.loadPatterns(patterns); } private function onPatternsLoaded (evt:Event) :void { this.patternLoader.removeEventListener(Event.INIT, this.onPatternsLoaded); this.bPatternsLoaded = true; this.checkForInitComplete(); } private function checkForInitComplete () :void { if (!this.bCameraParamsLoaded || !this.bPatternsLoaded || !this._flarSource) { return; } if (this.patternLoader.loadedPatterns.length == 0) { throw new Error("no markers successfully loaded."); } try { this.flarRaster = new FLARRgbRaster_BitmapData(this.flarSource.source); } catch (e:Error) { // this.flarSource not yet fully initialized this.flarRaster = null; } this.markerDetector = new FLARMultiMarkerDetector(this._cameraParams, this.patternLoader.loadedPatterns, this.patternLoader.unscaledMarkerWidths, this.patternLoader.loadedPatterns.length); //this.markerDetector.setContinueMode(true); if (this.thresholdSourceDisplay) { // if this.thresholdSourceDisplay was set to true before initialization of // this.flarSource and this.markerDetector, reset it. this.thresholdSourceDisplay = true; } this.smoother = new FLARMatrixSmoother_Average(); this.bInited = true; this.activate(); this.dispatchEvent(new Event(Event.INIT)); } //--------------------------------// } } internal class SourceInitStruct { internal var sourceWidth:int = 640; internal var sourceHeight:int = 480; internal var displayWidth:int = 640; internal var displayHeight:int = 480; internal var framerate:Number = 30; internal var downsampleRatio:Number = 0.5; internal var useProxy:Boolean = false; internal var loaderPath:String = ""; public function SourceInitStruct () {} }