Fine tune samples and configuration. Highlight matches in sample image.

Tutorials and examples of scripts and code
Post Reply
minx
Posts: 11
Joined: 24 Jan 2021, 20:51

Fine tune samples and configuration. Highlight matches in sample image.

Post by minx »

A simple script for looping files (needles) and searching a window (haystack) for matches.

Can be fine for fine-tuning parameters and image selections.

Purpose is for debugging and testing matches. Should be easy to both extend.

Not the most beautiful code :lol: but perhaps some find it fun to look at ...

Here is an example of result after searching with needles form this site:

Code: Select all

	[
		{fn: "img/t-01-qmark.png",  txt:  "qmark"},
		{fn: "img/t-02-lol.png",    txt:  "lol", opt: {
			sm: "coc",
			dpc: 2,
			sex: 1,
			conf: 87
		}},
		{fn: "img/t-03-preview.png", txt:  "preview"},
		{fn: "img/t-04-bell.png",    txt:  "bell"},
	],
Sample output.

The targets are marked with rectangles and crosses. (Questionmark, lol smiley, preview button and bell.)

Image

Code: Select all

/* vim:set noexpandtab tabstop=4 nosmarttab noshiftround shiftwidth=4 softtabstop=0: */
/* eslint no-unused-vars: off */
/* global Actionaz Algorithms Color Console
 * Notify Execution include Point ProcessHandle RawData Rect Script Size Stdio */
/* eslint-disable-next-line no-redeclare */
/* global Image Window */

const AA = Object.create(null);

// Simple logger for shell:
(function() {
	"use strict";
	const colors = {
		black: 31,
		red: 31,
		green: 32,
		yellow: 33,
		blue: 34,
		purple: 35,
		cyan: 36,
		white: 37
	};
	AA.print = function(str, color, bold) {
		var	s = str,
			c = colors[color],
			b = bold ? "1;" : ""
		;
		if (c)
			s = "\x1b[" + b + c + "m" + str + "\x1b[0m";
		else if (b)
			s = "\x1b[1m" + str + "\x1b[0m";

		Stdio.print(s);
	};
	AA.abort = function (msg, color) {
		if (msg)
			AA.print(msg + "\n", color || "purple", 1);
		AA.print("\nAborting. Bye!!\n", "red", 1);
		Execution.stop();
	};
	AA.dbg = function (str) {
		if (AA.debug)
			Stdio.print(";; " + str + "\n");
	};
	AA.note = function(title, txt, nosleep) {
		var myObject = new Notify({
			title: title,
			text: txt,
			timeout: 1000
		});
		myObject.show();
		// Notification closes when script exit
		if (!nosleep)
			Execution.sleep(3000);
	};
})();

// MAIN Routine:
(function(opt) {
	"use strict";

	// Locate window by title.
	// Try "tries" times before giving up.
	// If bool sleep is truth, sleep 1500ms between
	// each try.
	const get_window = function (title, tries, sleep) {
		var ret = null;
		--tries;
		if (sleep)
			Execution.sleep(1500);
		AA.dbg("get window[" + tries + "]: " + title);
		const win = Window.find({
			title: title,
			titleMode: Window.Wildcard
		});

		if (win.length > 1) {
			if (tries > 0)
				return get_window(title, tries, 1);
			AA.print("Found more then one window matching: " + title + "\n\n", "red", 1);
			for (var i = 0; i < win.length; ++i)
				Stdio.print(win[i].title() + "\n");
		} else if (win.length < 1) {
			if (tries > 0)
				return get_window(title, tries, 1);
			AA.print("Found NO window matching: " + title + "\n\n", "red", 1);
		} else if (win.length === 1) {
			ret = win[0];
		}
		return ret;
	};
	// Screenshot of win.
	// Save to fn_out
	const shoot = function (win, fn_out) {
		const shot = Image.takeScreenshot(win);
		if (!shot) {
			AA.print("Unable to shoot window\n", "red", 1);
			return null;
		}
		if (fn_out) {
			AA.dbg("Save shot as: " + fn_out);
			shot.saveToFile(fn_out);
		}
		return {win: win, shot: shot};
	};
	// Try to find Window specified by option:
	// 	opt.win_title
	// on success proceed with shot.
	const shoot_target = function (fn_out) {
		const win = get_window(opt.win_title, opt.tries);
		if (!win)
			return null;
		AA.dbg(win);
		return shoot(win, fn_out);
	};
	// win  Bring Window to foreground.
	// tw   Sleep tw milliseconds after request
	const win_to_fg = function (win, tw) {
		win.setForeground();
		Execution.sleep(tw ? tw : 300);
	};

	/*
	 * hay     Image image to search
	 * needle  Image image to search for
	 * opt     Obj   search options:
	 *     sm: search method. coc, crc or sd
	 *     conf: minimum confidence,
	 *     dpc : downPyramidCount
	 *     sex : search expansion
	 *     max : maximum matches
	 * */
	const IMG_SEARCH_METHOD = {
		coc: "CorrelationCoefficient",
		crc: "CrossCorrelation",
		sd : "SquaredDifference"
	};
	const find_subimages = function (hay, needle, opt) {
		opt = opt || {};
		var res = hay.findSubImages(needle, {
			method: IMG_SEARCH_METHOD[opt.sm || "coc"]
			, confidenceMinimum: opt.conf || 90
			, downPyramidCount: opt.dpc || 1
			, searchExpansion: opt.sex || 30
			, maximumMatches: opt.max || 100
		});
		return res;
	};

	// Draw horizontal line
	// img  Image
	// c    Color
	// x, y, w
	// d    Debug flag
	const draw_line_h = function (img, c, x, y, w, d) {
		const e = x + w;
		if (!d)
			AA.dbg("H: "
				+ x.toFixed(1) + ' x ' + y.toFixed(1) + ' - '
				+ e.toFixed(1) + ' x ' + y.toFixed(1) + ' '
				+ w
			);
		for ( ; x < e; ++x)
			img.setPixel(x, y, c);
	}
	// Draw vertical line
	const draw_line_v = function (img, c, x, y, h, d) {
		const e = y + h;
		if (!d)
			AA.dbg("V: "
				+ x.toFixed(1) + ' x ' + y.toFixed(1) + ' - '
				+ x.toFixed(1) + ' x ' + e.toFixed(1) + ' '
				+ h
			);
		for ( ; y < e; ++y)
			img.setPixel(x, y, c);
	}
	// Draw plus (+)
	const draw_plus = function (img, c, pt, spread) {
		spread = spread || 2;
		const z = spread * 2 + 1;
		draw_line_h (img, c, pt.sx - spread, pt.sy, z, 1);
		draw_line_v(img, c, pt.sx, pt.sy - spread, z, 1);
	}
	// Draw rectangle
	const draw_rect = function (img, c, r) {
		draw_line_h(img, c, r.x, r.y, r.w);
		draw_line_h(img, c, r.x, r.b, r.w);
		draw_line_v(img, c, r.x, r.y, r.h);
		draw_line_v(img, c, r.r, r.y, r.h);
	}
	// Combine various infoemation from match in sub image
	// search into an object.
	const match2rect = function (pos, sz, conf, f_info) {
		const r = {
			//x: Math.floor(pos.x - sz.width / 2),
			//y: Math.floor(pos.y - sz.height / 2),
			x: (pos.x - sz.width / 2),
			y: (pos.y - sz.height / 2),
			w: sz.width,
			h: sz.height,
			b: 0,
			r: 0,
			sx: pos.x,
			sy: pos.y,
			conf: conf,
			fn: f_info.fn,
			txt: f_info.txt
		};
		r.b = r.y + r.h;
		r.r = r.x + r.w;
		return r;
	}
	// Find sub-images in hay.
	// f_info is typically from options executing the script.
	const subimg2rect = function (hay, f_info) {
		const needle = new Image(f_info.fn);
		const match = find_subimages(hay, needle, f_info.opt);
		const ret = [];
		if (match) {
			match.forEach(function(m) {
				ret.push(match2rect(m.position, needle.size(), m.confidence, f_info));
			});
			return ret;
		}
		return null;
	}
	// Draw text: Require one to implement a methid for this
	// in the source code.
	/*
	const draw_txt = function (img, x, y, txt, opt) {
		img.setLabel(x, y, txt, opt);
	};
	*/
	const run = function () {
		// Locate and shoot target
		const target = shoot_target(opt.img.shot_saveas);
		if (!target)
			AA.abort();
		// the Image
		const hay = target.shot;
		// color for drawing
		const c = new Color();
		// matches
		var result = [];
		var tmp;
		// Loop needle stack from options. If found append obj with
		// data to result.
		opt.img.img.forEach(function(f) {
			if ((tmp = subimg2rect(hay, f)) !== null)
				result = result.concat(tmp);
		});
		AA.dbg(result.length + "\n" + result);
		// Loop and highlight finds.
		result.forEach(function(r) {
			// Rectangle by needle
			AA.dbg('Rect for: ' + r.fn);
			c.setNamedColor("#F00");
			draw_rect(hay, c, r);
			// A plus / ctross showing hit point
			c.setNamedColor("#15F");
			draw_plus(hay, c, r, Math.min(r.w, r.h) / 2);
			/* Draw text. Require implementation of methid in source code
			 * of actiona.
			draw_txt(hay, r.sx, r.sy, r.txt, {
				fontName: "Arial",
				fontSize: 18,
				bold: 81,
				italic: false,
				color: "#00A"
			});
			*/
		});
		// Save image with drawings to file
		hay.saveToFile(opt.img.dbg_saveas);
	};

	// More verbose logging to shell.
	AA.debug = 1;

	run();
})({
 /* *****************************************************************
  *
  *             O P T I O N S   A N D   I N P U T
  *
  * ******************************************************************
	// Number of tries locating target window
	tries: 3,
	// Title of target window
	win_title: "forum.jmgr.net - Post a new topic *",
	img: {
		// File OUT for original shot
		shot_saveas: "example-shot.jpg",
		// File OUT name for annotated image
		dbg_saveas : "example-out.png",
		// Array of images to search for.
		/*
		{
			fn  : file-path,
			opt : {
				sm  : search method. coc, crc or sd
				conf: minimum confidence,
				dpc : downPyramidCount
				sex : search expansion
				max : maximum matches
			},
			txt :	Text to write (label) matches with.
					Require one to implement a method for this in the
					actiona source code ...
		}
		*/

// ls -1 | sed 's/test-t\(.*\)-.*/{fn: "img\/test\/tuxes-\/\0", txt: "\1"},/'

		img: [
		{fn: "img/t-01-qmark.png", txt:  "qmark"},
		{fn: "img/t-02-lol.png", txt:  "lol", opt: {
			sm: "coc",
			dpc: 2,
			sex: 1,
			conf: 87
		}},
		{fn: "img/t-03-preview.png", txt:  "preview"},
		{fn: "img/t-04-bell.png", txt:  "bell"},
		],
	},
});

minx
Posts: 11
Joined: 24 Jan 2021, 20:51

Re: Fine tune samples and configuration. Highlight matches in sample image.

Post by minx »

If you want to add labels, one have to either use an external tool or modify the source code. At least from what I have found.

I have never written C++ code so use this with caution :wink: , but add if you want to test.

The code below adds a function to Image:
  • Image.setLabel(int x, int y, str label[, obj options])
x and y are position. label is the text and options is an optional object with font settings:
  • str fontName: font to use, e.g. Arial, Courier
  • int fontSize: font size
  • int bold: 0 - 99
  • bool italic: (true / false)
  • str color: color of the text, e.g "#FAFAFA"
Example:

Code: Select all

myImage.setLabel(13, 46, "foo", { fontName: "Arial", fontSize: 46, color: "#F00" });
======================================================================
The C++ code
======================================================================

image.c

In image.c add two methods:

Code: Select all

	QScriptValue Image::setLabel(int x, int y, const QString &label, const QScriptValue &options) {
		QPainter painter(&mImage);
		int fontSize;
		int bold;
		bool italic;
		QString fontName;
		QColor color;
		QPen pen;

		findFontOptions(options, &fontSize, &bold, &italic, &fontName, &color);

		painter.setRenderHint(QPainter::Antialiasing);
		painter.setFont(QFont(fontName, fontSize, bold, italic));
		// QPen penHLines(QColor("#0e6a65"), 9, Qt::DotLine, Qt::FlatCap, Qt::RoundJoin);
		pen.setBrush(color);

		painter.setPen(pen);
		painter.drawText(QPoint(x, y), label);

		return thisObject();
	}
	void Image::findFontOptions(
			const QScriptValue &options,
			int *fontSize,
			int *bold,
			bool *italic,
			QString *fontName,
			QColor *color) const {

		QScriptValueIterator it(options);

		if (fontSize)
			*fontSize = 14;
		if (bold)
			*bold = 50;
		if (italic)
			*italic = false;
		if (fontName)
			*fontName = tr("Arial");
		if (color)
			*color = QColor("#F00");

		while(it.hasNext()) {
			it.next();
			qDebug() << it.name() << ": " << it.value().toString();
			if (fontSize && it.name() == QStringLiteral("fontSize"))
				*fontSize = it.value().toInt32();
			if (bold && it.name() == QStringLiteral("bold"))
				*bold = it.value().toInt32();
			if (italic && it.name() == QStringLiteral("italic"))
				*italic = it.value().toBool();
			if (fontName && it.name() == QStringLiteral("fontName"))
				*fontName = it.value().toString();
			if (color && it.name() == QStringLiteral("color")) {
				QObject *object = it.value().toQObject();
				if (auto codeColor = qobject_cast<Color*>(object))
					*color = QColor(codeColor->color());
				else
					*color = QColor(it.value().toString());
			}
		}
	}
----------------------------------------------------------------------------------------------------

image.h

In image.h add:

Code: Select all

		QScriptValue setLabel(int x, int y, const QString &label, const QScriptValue &options = QScriptValue());
under public slots:

And

Code: Select all

	void findFontOptions(
			const QScriptValue &options,
			int *fontSize,
			int *bold,
			bool *italic,
			QString *fontName,
			QColor *color) const;
under private:

Post Reply