database - Image retrieval system by Colour from the web using C++ with openframeworks

Question

I am writing a program in C++ and openFrameworks that should hopefully implement an image retrieval system by colour matching. I have got an algorithm to find the match in a database by an rgb value. For example, if I have a database of 1000 pictures on my computer and I have a query rgb value 255,0,0 the program would look through 1000 pictures and find the closest match. However, my problem is that I want it to also look for the match on the web. I have been trying to find how to get images from websites, however, if you don't know the specific url of the image it's hard to get hold of the data. Maybe somebody has got some knowledge of how to get hold of images on websites? Ideally, the program would go on specified website and search through every webpage for the images, it would then compare each image to the query and output the closest match.

Answer 1

As I mentioned in my comment, it's a matter of converting from RGB colourspace to Lab* colourspace and using the euclidean distance to the average colour of the image from the database.

Here's a basic demo:

#include "testApp.h"

//ported from http://cookbooks.adobe.com/post_Useful_color_equations__RGB_to_LAB_converter-14227.html
struct Color{
    float R,G,B,X,Y,Z,L,a,b;
};

#define REF_X 95.047; // Observer= 2°, Illuminant= D65
#define REF_Y 100.000;
#define REF_Z 108.883;

Color rgb2xyz(int R,int G,int B){
    float r = R / 255.0;
    float g = G / 255.0;
    float b = B / 255.0;

    if (r > 0.04045){ r = pow((r + 0.055) / 1.055, 2.4); }
    else { r = r / 12.92; }
    if ( g > 0.04045){ g = pow((g + 0.055) / 1.055, 2.4); }
    else { g = g / 12.92; }
    if (b > 0.04045){ b = pow((b + 0.055) / 1.055, 2.4); }
    else {  b = b / 12.92; }

    r = r * 100;
    g = g * 100;
    b = b * 100;
    //Observer. = 2°, Illuminant = D65
    Color xyz;
    xyz.X = r * 0.4124 + g * 0.3576 + b * 0.1805;
    xyz.Y = r * 0.2126 + g * 0.7152 + b * 0.0722;
    xyz.Z = r * 0.0193 + g * 0.1192 + b * 0.9505;
    return xyz;
}
Color xyz2lab(float X,float Y, float Z){
    float x = X / REF_X;
    float y = Y / REF_X;
    float z = Z / REF_X;

    if ( x > 0.008856 ) { x = pow( x , .3333333333f ); }
    else { x = ( 7.787 * x ) + ( 16/116.0 ); }
    if ( y > 0.008856 ) { y = pow( y , .3333333333f ); }
    else { y = ( 7.787 * y ) + ( 16/116.0 ); }
    if ( z > 0.008856 ) { z = pow( z , .3333333333f ); }
    else { z = ( 7.787 * z ) + ( 16/116.0 ); }

    Color lab;
    lab.L = ( 116 * y ) - 16;
    lab.a = 500 * ( x - y );
    lab.b = 200 * ( y - z );
    return lab;
}
Color lab2xyz(float l, float a, float b){
    float y = (l + 16) / 116;
    float x = a / 500 + y;
    float z = y - b / 200;

    if ( pow( y , 3 ) > 0.008856 ) { y = pow( y , 3 ); }
    else { y = ( y - 16 / 116 ) / 7.787; }
    if ( pow( x , 3 ) > 0.008856 ) { x = pow( x , 3 ); }
    else { x = ( x - 16 / 116 ) / 7.787; }
    if ( pow( z , 3 ) > 0.008856 ) { z = pow( z , 3 ); }
    else { z = ( z - 16 / 116 ) / 7.787; }

    Color xyz;
    xyz.X = x * REF_X;
    xyz.Y = y * REF_Y;
    xyz.Z = z * REF_Z;
    return xyz;
}
Color xyz2rgb(float X,float Y,float Z){
    //X from 0 to  95.047      (Observer = 2°, Illuminant = D65)
    //Y from 0 to 100.000
    //Z from 0 to 108.883
    X = ofClamp(X, 0, 95.047);

    float x = X * .01;
    float y = Y * .01;
    float z = Z * .01;

    float r = x * 3.2406 + y * -1.5372 + z * -0.4986;
    float g = x * -0.9689 + y * 1.8758 + z * 0.0415;
    float b = x * 0.0557 + y * -0.2040 + z * 1.0570;

    if ( r > 0.0031308 ) { r = 1.055 * pow( r , ( 1 / 2.4f ) ) - 0.055; }
    else { r = 12.92 * r; }
    if ( g > 0.0031308 ) { g = 1.055 * pow( g , ( 1 / 2.4f ) ) - 0.055; }
    else { g = 12.92 * g; }
    if ( b > 0.0031308 ) { b = 1.055 * pow( b , ( 1 / 2.4f ) ) - 0.055; }
    else { b = 12.92 * b; }

    Color rgb;
    rgb.R = round( r * 255 );
    rgb.G = round( g * 255 );
    rgb.B = round( b * 255 );
    return rgb;
}
Color rgb2lab(int R,int G,int B){
    Color xyz = rgb2xyz(R, G, B);
    return xyz2lab(xyz.X, xyz.Y, xyz.Z);
}
Color lab2rgb(int L,int a,int b){
    Color xyz = lab2xyz(L, a, b);
    return xyz2rgb(xyz.X, xyz.Y, xyz.Z);
}

Color getAverage(ofImage img){
    Color avg;
    avg.L = avg.a = avg.b = 0;

    int total = img.width * img.height;
    for(int y = 0 ; y < img.height; y++){
        for(int x = 0 ; x < img.width; x++){
            ofColor c = img.getColor(x, y);
            Color lab = rgb2lab(c.r,c.g,c.b);
            avg.L += lab.L;
            avg.a += lab.a;
            avg.b += lab.b;
        }
    }

    avg.L /= total;
    avg.a /= total;
    avg.b /= total;
    return avg;
}
ofImage images[6];
Color   averages[6];
ofColor averagesRGB[6];

ofImage colorPicker;
ofColor searchClr;

int closestId = -1;

//--------------------------------------------------------------
void testApp::setup(){
    colorPicker.loadImage("colormap.gif");

    images[0].loadImage("red.jpg");
    images[1].loadImage("green.jpg");
    images[2].loadImage("blue.jpg");
    images[3].loadImage("cyan.jpg");
    images[4].loadImage("magenta.jpg");
    images[5].loadImage("yellow.jpg");

    for(int i = 0 ;  i < 6; i++){
        averages[i] = getAverage(images[i]);
        Color avgRGB = lab2rgb(averages[i].L, averages[i].a, averages[i].b);
        averagesRGB[i] = ofColor(avgRGB.R,avgRGB.G,avgRGB.B);
    }

}

//--------------------------------------------------------------
void testApp::update(){
    //pick a colour
    searchClr = colorPicker.getColor(mouseX,mouseY-500);
    //find closest - might want to that on an event
    Color searchLab = rgb2lab(searchClr.r, searchClr.g, searchClr.b);
    float minDist = 10000000;
    for(int i = 0 ; i < 6; i++){
        Color Lab = averages[i];
        float dL = Lab.L - searchLab.L;
        float da = Lab.a - searchLab.a;
        float db = Lab.b - searchLab.b;
        float dist = sqrt(dL*dL + da*da + db*db);
        if(dist < minDist){
            minDist = dist;
            closestId = i;
        }
    }
}

//--------------------------------------------------------------
void testApp::draw(){
    for(int i = 0 ;  i < 6; i++){
        //indexed image
        images[i].draw(images[i].width * i, 0);
        //average colour
        ofPushStyle();
        ofSetColor(averagesRGB[i]);
        ofRect(images[i].width * i, images[i].height, images[i].width, images[i].width);
        ofPopStyle();
    }
    ofPushStyle();
    ofSetColor(searchClr);
    ofRect(200,500,200,200);
    ofPopStyle();
    colorPicker.draw(0,500);
    if(closestId >= 0){
        images[closestId].draw(400, 500);
    }
}

//--------------------------------------------------------------
void testApp::keyPressed(int key){

}

//--------------------------------------------------------------
void testApp::keyReleased(int key){

}

//--------------------------------------------------------------
void testApp::mouseMoved(int x, int y){

}

//--------------------------------------------------------------
void testApp::mouseDragged(int x, int y, int button){

}

//--------------------------------------------------------------
void testApp::mousePressed(int x, int y, int button){

}

//--------------------------------------------------------------
void testApp::mouseReleased(int x, int y, int button){

}

//--------------------------------------------------------------
void testApp::windowResized(int w, int h){

}

//--------------------------------------------------------------
void testApp::gotMessage(ofMessage msg){

}

//--------------------------------------------------------------
void testApp::dragEvent(ofDragInfo dragInfo){ 

}

The coding style isn't brilliant but it's just to illustrate the idea. Of course you would need to load the images from the url first and index the average colour in Lab* for each in a database (vector at runtime or otherwise). The above code is also available as an Xcode project