image searching

3 files changed, 261 insertions, 6 deletions

diff --git a/offsetProject/src/imageStore.h b/offsetProject/src/imageStore.h
index 1db0ee2..5f30163 100644
--- a/offsetProject/src/imageStore.h
+++ b/offsetProject/src/imageStore.h
@@ -24,6 +24,10 @@ class imageStore : public ofThread{
 		imageStore(){
 			instagram_url = "https://api.instagram.com/v1/tags/tycleeson/media/recent?client_id=c1d74d701fdf4ddd9f8d30ee9e8f944b";
 			interval=5.00f;
+			ofImage img;
+			img.allocate(MAX_TILE_SIZE,MAX_TILE_SIZE,OF_IMAGE_COLOR);
+			images["000000"]=img;
+			colours["000000"]=ofColor(0,0,0);
 		}
 
 		void set_interval(float _interval){
@@ -38,6 +42,92 @@ class imageStore : public ofThread{
             stopThread();
         }
 
+        //naive implementation
+        //too slow!
+        //looks shit!
+
+        //how to make the search algorithm faster and better
+        //http://www.semanticmetadata.net/lire/
+
+     	/*
+		1. The box nature of the image matching isn't appealing
+		2. The way that the images flip quickly isn't appealing
+
+		can we improve this by changing the screen gradually in some way?
+
+		search tree/ octree
+
+		each node associates an image, start with a black image
+
+		a node owns a volume of RGB space
+
+		new image: determine which octant it lands in
+		subdivide the octant in R, G or B and give half to each image
+		
+		Q: does this lead to the possibility that images end up in the wrong place?
+
+		with the naive algorithm, we measure distance between the colours in RGB space
+
+		it becomes a long process because the number of calculations increases exponentially
+
+		space partitioning: we observe that the image will often be nearest to an image in the same box
+		although it may not be if they are at the edges
+
+		so at each level, each node of the tree contains a list of images
+
+		we traverse the tree and try to find the lowest level box with a match
+		if the lowest level box with a match has more than 1 we compute distance?
+
+		a difference algorithm
+		minimise the error
+		this involves comparing every pixel though
+
+		gpu?
+
+		put every quarter of every image into an image tree
+
+		every image has an entry on levels 1-8
+		start at level 8 (most detailed)
+		check the leaf node ie 10110101 01101001 00110101
+		if there are 1 or more images here choose one
+		if none go to the next level...
+
+		ie 1011010 0110100 0011010
+		if there are more than 1 images here find the nearest
+		if there is 1 choose it
+		if there are none go to the next level
+
+     	*/
+
+
+        std::map<std::string,ofColor> colours;
+        ofImage& get_image(const ofColor& col){
+        	//float shortest_dist=999999.0f;
+        	int sd=1000;
+        	ofImage& im=images.begin()->second;
+        	std::string s=images.begin()->first;
+        	for (map<string,ofImage>::iterator it=images.begin();it!=images.end();++it){
+        		ofColor& c=colours[it->first];
+        		int rd=c.v[0]-col.v[0];
+        		int gd=c.v[1]-col.v[1];
+        		int bd=c.v[2]-col.v[2];
+        		//float dist=pow((float)((rd*rd)+(gd*gd)+(bd*bd)),0.5);
+        		int dist=abs(rd)+abs(gd)+abs(bd);
+        		if (dist<sd){
+        			sd=dist;
+        			im=it->second;
+        			s=it->first;
+        		}
+        	}
+        	//cerr<<"got image "<<s<<endl;
+        	return im;
+        }
+        ofColor get_colour(const ofImage& _img){
+        	ofImage img=_img;
+        	img.resize(1,1);
+        	return ofColor(img.getPixels()[0],img.getPixels()[1],img.getPixels()[2]);
+        }
+
 		//--------------------------
 		void threadedFunction(){
 
@@ -54,7 +144,9 @@ class imageStore : public ofThread{
 					img.loadImage(ofToString(IMAGE_STORE_SIZE)+"/"+image_path.getFiles()[i].getFileName());
 					img.resize(MAX_TILE_SIZE,MAX_TILE_SIZE);
 					if( lock() ){
+						colours[image_path.getFiles()[i].getBaseName()]=get_colour(img);
 						images[image_path.getFiles()[i].getBaseName()]=img;
+						cerr<<image_path.getFiles()[i].getBaseName()<<": "<<colours[image_path.getFiles()[i].getBaseName()]<<endl;
 						to_update.push_back(image_path.getFiles()[i].getBaseName());
 						unlock();
 					}
@@ -92,6 +184,7 @@ class imageStore : public ofThread{
 							img.saveImage(ofToString(IMAGE_STORE_SIZE)+"/"+id+".png");
 							img.resize(MAX_TILE_SIZE,MAX_TILE_SIZE);
 							if( lock() ){
+								colours[id]=get_colour(img);
 								images[id]=img;
 								to_update.push_back(id);
 								unlock();
@@ -103,8 +196,8 @@ class imageStore : public ofThread{
 			}
 		}
 
-		//--------------------------
-		void draw(){
+		void update(){
+			//loads one texture
 			if( lock() ){
 				if (to_update.size()){
 					std::string im = to_update.front();
@@ -129,6 +222,13 @@ class imageStore : public ofThread{
 					//}
 					//cout<<"loaded "<<im<<" "<<ofToLong(im)%(long)(ofGetWidth()-images[im].getWidth()+1)<<","<<ofToLong(im)%(long)(ofGetHeight()-images[im].getHeight()+1)<<endl;
 				}
+				unlock();
+			}
+		}
+
+		//--------------------------
+		void draw(){
+			if( lock() ){
 				/*
 				for (map<string,ofImage>::iterator i=images.begin();i!=images.end();++i){
 					if(i->second.isUsingTexture()){
diff --git a/offsetProject/src/ofApp.cpp b/offsetProject/src/ofApp.cpp
index f65746e..7c5428c 100644
--- a/offsetProject/src/ofApp.cpp
+++ b/offsetProject/src/ofApp.cpp
@@ -60,6 +60,17 @@ void ofApp::setup() {
 		
 	ofSetFrameRate(60);
 
+	kinect.setRegistration(true); 
+	kinect.init();
+	kinect.open();
+	colourImage.allocate(kinect.width, kinect.height);
+	depthImage.allocate(kinect.width, kinect.height);
+	farThreshold = 70;
+	angle = 0;
+	kinect.setCameraTiltAngle(angle);
+	kinect.enableDepthNearValueWhite(true);
+	kinect.setDepthClipping(0,4000);
+
 	store.start();
 	
 	mode=MODE_COLOURTILES;
@@ -72,6 +83,51 @@ void ofApp::update() {
 	
 	ofSetWindowTitle(ofToString(ofGetFrameRate()));
 	
+	kinect.update();
+	
+	// there is a new frame and we are connected
+	if(kinect.isFrameNew()) {
+		
+		depthImage.setFromPixels(kinect.getDepthPixels(), kinect.width, kinect.height);
+		colourImage.setFromPixels(kinect.getPixels(), kinect.width, kinect.height);
+		
+		depthImage.threshold(farThreshold);
+
+		//threshold needs to be multiplied by the original
+		depthImage2.setFromPixels(kinect.getDepthPixels(), kinect.width, kinect.height);
+		depthImage*=depthImage2;
+
+		//depthImage.extend(extend_w,extend_h);
+		//colourImage.extend(extend_w,extend_h);
+
+		int h=ceil(ofGetHeight()/MAX_TILE_SIZE);
+		int w=(h*4)/3;
+		depthImage.resize(w,h);
+		colourImage.resize(w,h);
+		
+		// mark pixels and texture dirty
+		depthImage.flagImageChanged();
+		colourImage.flagImageChanged();
+
+		/*
+		ofxCvColorImage *prevCol=&colourImage;
+		ofxCvGrayscaleImage *prevDepth=&depthImage;
+
+
+		for (int i=0;i<colourTiles.size();i++){
+			colourTiles[i].scaleIntoMe(*prevCol);
+			depthTiles[i].scaleIntoMe(*prevDepth);
+			prevCol=&colourTiles[i];
+			prevDepth=&depthTiles[i];
+
+			colourTiles[i].flagImageChanged();
+			depthTiles[i].flagImageChanged();
+		}
+		*/
+	}
+
+	store.update();
+	
 	
 }
 
@@ -84,7 +140,39 @@ void ofApp::draw() {
 
 	ofSetColor(255, 255, 255);
 
-	store.draw();
+	int w,xstart,ystart,kinexstart;
+
+	switch(mode){
+		case MODE_COLOURTILES:
+			//store.draw();
+			xstart=(ofGetWidth()/2)-((colourImage.getHeight()/2)*MAX_TILE_SIZE);
+			ystart=(ofGetHeight()/2)-((colourImage.getHeight()/2)*MAX_TILE_SIZE);
+			kinexstart=(colourImage.getWidth()/2)-(colourImage.getHeight()/2);
+			ofPushMatrix();
+				ofTranslate(xstart,ystart);
+				w=colourImage.getWidth();
+				for (int i=0;i<depthImage.getHeight();i++){
+					for (int j=0;j<depthImage.getHeight();j++){
+						uint8_t* ppt=&colourImage.getPixels()[(j*w+(w-(i+kinexstart)))*3];
+						store.get_image(ofColor(ppt[0],ppt[1],ppt[2])).draw(i*MAX_TILE_SIZE,j*MAX_TILE_SIZE);
+						//ofSetColor(ppt[0],ppt[1],ppt[2]);
+						//ofRect(i*MAX_TILE_SIZE,j*MAX_TILE_SIZE,MAX_TILE_SIZE,MAX_TILE_SIZE);
+					}
+				}
+			ofPopMatrix();
+			break;
+
+		case MODE_DEPTH:
+
+			depthImage.draw(0,0,ofGetWidth(),ofGetHeight());
+			break;
+
+		case MODE_COMPONENTS:
+
+			depthImage.draw(0,0, 640,480);
+			colourImage.draw(640,0, 640,480);
+			break;
+	}
     
 }
 
@@ -100,10 +188,72 @@ void ofApp::exit() {
 void ofApp::keyPressed (int key) {
 	switch (key) {
 			
+		case '>':
+		case '.':
+			farThreshold ++;
+			if (farThreshold > 255) farThreshold = 255;
+			break;
+			
+		case '<':
+		case ',':
+			farThreshold --;
+			if (farThreshold < 0) farThreshold = 0;
+			break;
+			
+			
+		case 'w':
+			kinect.enableDepthNearValueWhite(!kinect.isDepthNearValueWhite());
+			break;
+			
+		case 'o':
+			kinect.setCameraTiltAngle(angle); // go back to prev tilt
+			kinect.open();
+			break;
+			
+		case 'c':
+			kinect.setCameraTiltAngle(0); // zero the tilt
+			kinect.close();
+			break;
+			
+		case '1':
+			kinect.setLed(ofxKinect::LED_GREEN);
+			break;
+			
+		case '2':
+			kinect.setLed(ofxKinect::LED_YELLOW);
+			break;
+			
+		case '3':
+			kinect.setLed(ofxKinect::LED_RED);
+			break;
+			
+		case '4':
+			kinect.setLed(ofxKinect::LED_BLINK_GREEN);
+			break;
+			
+		case '5':
+			kinect.setLed(ofxKinect::LED_BLINK_YELLOW_RED);
+			break;
+			
+		case '0':
+			kinect.setLed(ofxKinect::LED_OFF);
+			break;
+			
+		case OF_KEY_UP:
+			angle++;
+			if(angle>30) angle=30;
+			kinect.setCameraTiltAngle(angle);
+			break;
+			
+		case OF_KEY_DOWN:
+			angle--;
+			if(angle<-30) angle=-30;
+			kinect.setCameraTiltAngle(angle);
+			break;
 		
 		case OF_KEY_LEFT:
 			mode--;
-			if (mode<0) mode==NUM_MODES-1;
+			if (mode<0) mode=NUM_MODES-1;
 			break;
 
 		case OF_KEY_RIGHT:
diff --git a/offsetProject/src/ofApp.h b/offsetProject/src/ofApp.h
index f1f8253..a718aa1 100644
--- a/offsetProject/src/ofApp.h
+++ b/offsetProject/src/ofApp.h
@@ -13,7 +13,7 @@
 #define NUM_MODES 3
 
 
-
+//do we need the extend thing?
 
 class ofApp : public ofBaseApp {
 public:
@@ -30,7 +30,12 @@ public:
 	void windowResized(int w, int h);
 
 	imageStore store;
-	int mode;
+
+	int mode,farThreshold,angle;
+
+	ofxCvColorImage colourImage;
+	ofxCvGrayscaleImage depthImage,depthImage2;
+	ofxKinect kinect;
 
 	bool fullscreen;
 };