Android TensorFlow Lite Object Recognition

About Face Recognition, many sdks are currently available on the internet, but not free.

TensorFlow Lite is a free and lightweight library.

This article used object recognition as example.

Reference

Step 1: Download TensorFlow Model

Use MobileNet_v1_1.0_224 Model Link

After unzipping, we get the file : mobilenet_v1_1.0_224.tflite.

Step 2: Dwonload Label

MobileNet_v1_1.0_224 Model has 1001 type, but it doesn’t have tags.
We can download the classified tags from this. Link

Step 3: Import TensorFlow Lite

Add libraries at dependencies in build.gradle.

build.gradle

dependencies {
    ...
    implementation 'com.github.bumptech.glide:glide:4.3.1'
    implementation 'org.tensorflow:tensorflow-lite:0.0.0-nightly'
}

Inset the text at android to avoid compress the tensor lite model.

build.gradle

android {
    ...
    aaptOptions {
        noCompress "tflite"
    }
}

Step 4: Add Model

Create assets folder in main folder, and copy mobilenet_v1_1.0_224.tflite and label .txt at here.

Designed two button : Import model and analysis image.

The source code for import model :

loadModeFile : transfer mode file to ByteBuffer，than initialize Interpreter.
load_mode : Use Interpreter to analysis image，and setup the thread length (tflite.setNumThreads)

MainActivity.java

private final String MODEL_NAME = "mobilenet_v1_1.0_224";

private Interpreter tflite = null;

@Override
public void onClick(View v) {
	switch(v.getId()) {
		case R.id.load_model:
			 load_model(MODEL_NAME);
		break;
                ...
	 }
}

private void load_model(String model) {
	try {
		tflite = new Interpreter(loadModelFile(model));
		Toast.makeText(MainActivity.this, model + " model load success", Toast.LENGTH_SHORT).show();
		Log.d(TAG, model + " model load success");
		tflite.setNumThreads(4);
		load_result = true;
	} catch (IOException e) {
		Toast.makeText(MainActivity.this, model + " model load fail", Toast.LENGTH_SHORT).show();
		Log.d(TAG, model + " model load fail");
		load_result = false;
		e.printStackTrace();
	}
}

private ByteBuffer loadModelFile(String model) throws IOException {
	AssetFileDescriptor fileDescriptor = getApplicationContext().getAssets().openFd(model + ".tflite");
	FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
	FileChannel fileChannel = inputStream.getChannel();
	long startOffset = fileDescriptor.getStartOffset();
	long declaredLength = fileDescriptor.getDeclaredLength();
	return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
}

Step 5: Insert Label

Read label file in onCreate function.

MainActivity.java

@Override
protected void onCreate(Bundle savedInstanceState) {
	super.onCreate(savedInstanceState);
	...
	readCacheLabelFromLocalFile();
}

private void readCacheLabelFromLocalFile() {
	try {
		AssetManager assetManager = getApplicationContext().getAssets();
		BufferedReader reader = new BufferedReader(new InputStreamReader(assetManager.open(LABEL_NAME + ".txt")));
		String readLine = null;
		while ((readLine = reader.readLine()) != null) {
			resultLabel.add(readLine);
		}
		reader.close();
	} catch (Exception e) {
		Log.e("labelCache", "error " + e);
	}
}

Step 6: Analysis Image

predict_image ：Zipping image first, than transfer image to ByteBuffer.
Using Interpreter.run() to analysis.

get_max_result : Get a classified label with highest probability.

MainActivity.java

private List<String> resultLabel = new ArrayList<>();
private ImageView mImage;
private int[] ddims = {1, 3, 224, 224};
private TextView mResult;

@Override
public void onClick(View v) {
	switch(v.getId()) {
		...
		case R.id.use_photo:
                
			RequestOptions options = new RequestOptions().skipMemoryCache(true).diskCacheStrategy(DiskCacheStrategy.NONE);
			File file = new File("/storage/sdcard1/mouse.jpeg");
			Uri photoUri = Uri.fromFile(file);
			Glide.with(MainActivity.this).load(photoUri).apply(options).into(mImage);
			predict_image(file.getAbsolutePath());
			break;
		default:
			break;
	}
}

private void predict_image(String image_path) {
	// picture to float array
	Bitmap bmp = getScaleBitmap(image_path);
	ByteBuffer inputData = getScaledMatrix(bmp, ddims);
	try {
		float[][] labelProbArray = new float[1][1001];
		long start = System.currentTimeMillis();
		// get predict result
		tflite.run(inputData, labelProbArray);
		long end = System.currentTimeMillis();	
		long time = end - start;

		float[] results = new float[labelProbArray[0].length];
		System.arraycopy(labelProbArray[0], 0, results, 0, labelProbArray[0].length);
		// show predict result and time
		int r = get_max_result(results);
            
		String show_text = "result：" + r + "\nname：" + resultLabel.get(r) + "\nprobability：" + results[r] + "\ntime：" + time + "ms";
		mResult.setText(show_text);
	} catch (Exception e) {
		e.printStackTrace();
	}
}

private Bitmap getScaleBitmap(String filePath) {
	BitmapFactory.Options opt = new BitmapFactory.Options();
	opt.inJustDecodeBounds = true;
	BitmapFactory.decodeFile(filePath, opt);

	int bmpWidth = opt.outWidth;
	int bmpHeight = opt.outHeight;
	int maxSize = 500;
	// compress picture with inSampleSize
	opt.inSampleSize = 1;
	while (true) {
		if (bmpWidth / opt.inSampleSize < maxSize || bmpHeight / opt.inSampleSize < maxSize) {
		break;
		}
		opt.inSampleSize *= 2;
	}
	opt.inJustDecodeBounds = false;

	return BitmapFactory.decodeFile(filePath, opt);
}

private ByteBuffer getScaledMatrix(Bitmap bitmap, int[] ddims) {
	ByteBuffer imgData = ByteBuffer.allocateDirect(ddims[0] * ddims[1] * ddims[2] * ddims[3] * 4);
	imgData.order(ByteOrder.nativeOrder());
	// get image pixel
	int[] pixels = new int[ddims[2] * ddims[3]];
	Bitmap bm = Bitmap.createScaledBitmap(bitmap, ddims[2], ddims[3], false);
	bm.getPixels(pixels, 0, bm.getWidth(), 0, 0, ddims[2], ddims[3]);
	int pixel = 0;
	for (int i = 0; i < ddims[2]; ++i) {
		for (int j = 0; j < ddims[3]; ++j) {
			final int val = pixels[pixel++];
			imgData.putFloat(((((val >> 16) & 0xFF) - 128f) / 128f));
			imgData.putFloat(((((val >> 8) & 0xFF) - 128f) / 128f));
			imgData.putFloat((((val & 0xFF) - 128f) / 128f));
		}
	}

	if (bm.isRecycled()) {
		bm.recycle();
	}
	return imgData;
}

private int get_max_result(float[] result) {
	float probability = result[0];
	int r = 0;
	for (int i = 0; i < result.length; i++) {
		if (probability < result[i]) {
			probability = result[i];
			r = i;
		}
	}
	return r;
}

Result