Added files except model files, model is too big

app.py

@@ -0,0 +1,151 @@
+import cv2
+import numpy as np
+from pil import Image
+import streamlit as st
+import tensorflow as tf
+from tensorflow.keras.models import load_model
+
+# most of this code has been obtained from Daature's prediction script
+# https://github.com/datature/resources/blob/main/scripts/bounding_box/prediction.py
+
+st.set_option('deprecation.showfileUploaderEncoding', False)
+
+@st.cache(allow_output_mutation=True)
+def load_model():
+	return tf.saved_model.load('./saved_model')
+
+def load_label_map(label_map_path):
+    """
+    Reads label map in the format of .pbtxt and parse into dictionary
+    Args:
+      label_map_path: the file path to the label_map
+    Returns:
+      dictionary with the format of {label_index: {'id': label_index, 'name': label_name}}
+    """
+    label_map = {}
+
+    with open(label_map_path, "r") as label_file:
+        for line in label_file:
+            if "id" in line:
+                label_index = int(line.split(":")[-1])
+                label_name = next(label_file).split(":")[-1].strip().strip('"')
+                label_map[label_index] = {"id": label_index, "name": label_name}
+    return label_map
+	
+def predict_class(image, model):
+	image = tf.cast(image, tf.float32)
+	image = tf.image.resize(image, [150, 150])
+	image = np.expand_dims(image, axis = 0)
+	return model.predict(image)
+
+def plot_boxes_on_img(color_map, classes, bboxes, image_origi, origi_shape):
+	for idx, each_bbox in enumerate(bboxes):
+		color = color_map[classes[idx]]
+
+		## Draw bounding box
+		cv2.rectangle(
+			image_origi,
+			(int(each_bbox[1] * origi_shape[1]),
+			 int(each_bbox[0] * origi_shape[0]),),
+			(int(each_bbox[3] * origi_shape[1]),
+			 int(each_bbox[2] * origi_shape[0]),),
+			color,
+			2,
+		)
+		## Draw label background
+		cv2.rectangle(
+			image_origi,
+			(int(each_bbox[1] * origi_shape[1]),
+			 int(each_bbox[2] * origi_shape[0]),),
+			(int(each_bbox[3] * origi_shape[1]),
+			 int(each_bbox[2] * origi_shape[0] + 15),),
+			color,
+			-1,
+		)
+		## Insert label class & score
+		cv2.putText(
+			image_origi,
+			"Class: {}, Score: {}".format(
+				str(category_index[classes[idx]]["name"]),
+				str(round(scores[idx], 2)),
+			),
+			(int(each_bbox[1] * origi_shape[1]),
+			 int(each_bbox[2] * origi_shape[0] + 10),),
+			cv2.FONT_HERSHEY_SIMPLEX,
+			0.3,
+			(0, 0, 0),
+			1,
+			cv2.LINE_AA,
+		)
+	return image_origi
+
+
+# Webpage code starts here
+
+st.title('Banana ripeness detection 🍌')
+st.text('made by XXX') #TODO change with your name
+st.markdown('## Find out if a banana is too ripe!')
+
+with st.spinner('Model is being loaded...'):
+	model = load_model()
+
+# ask user to upload an image
+file = st.file_uploader("Upload an image of a banana", type=["jpg", "png"])
+
+if file is None:
+	st.text('Waiting for upload...')
+else:
+	st.text('Running inference...')
+	# open image
+	test_image = Image.open(file).convert("RGB")
+	origi_shape = np.asarray(test_image).shape
+	# resize image to default shape
+	default_shape = 320
+	image_resized = np.array(test_image.resize((default_shape, default_shape)))
+
+	## Load color map
+	category_index = load_label_map("./label_map.pbtxt")
+	# color of each label. check label_map.pbtxt to check the index for each class
+	# TODO Add more colors if there are more classes
+	color_map = {
+		2: [255, 0, 0], # overripe -> red
+		1: [0, 255, 0] # ripe -> green
+	}
+
+	## The model input needs to be a tensor
+	input_tensor = tf.convert_to_tensor(image_resized)
+	## The model expects a batch of images, so add an axis with `tf.newaxis`.
+	input_tensor = input_tensor[tf.newaxis, ...]
+
+	## Feed image into model and obtain output
+	detections_output = model(input_tensor)
+	num_detections = int(detections_output.pop("num_detections"))
+	detections = {key: value[0, :num_detections].numpy() for key, value in detections_output.items()}
+	detections["num_detections"] = num_detections
+
+	## Filter out predictions below threshold
+	# if threshold is higher, there will be fewer predictions
+	# TODO change this number to see how the predictions change
+	confidence_threshold = 0.5
+	indexes = np.where(detections["detection_scores"] > confidence_threshold)
+
+	## Extract predicted bounding boxes
+	bboxes = detections["detection_boxes"][indexes]
+	# there are no predicted boxes
+	if len(bboxes) == 0:
+		st.error('No boxes predicted')
+	# there are predicted boxes
+	else:
+		st.success('Boxes predicted')
+		classes = detections["detection_classes"][indexes].astype(np.int64)
+		scores = detections["detection_scores"][indexes]
+
+		# plot boxes and labels on image
+		image_origi = np.array(Image.fromarray(image_resized).resize((origi_shape[1], origi_shape[0])))
+		image_origi = plot_boxes_on_img(color_map, classes, bboxes, image_origi, origi_shape)
+
+		# show image in web page
+		st.image(Image.fromarray(image_origi), caption="Image with predictions", width=400)
+		st.markdown("### Predicted boxes")
+		for idx in range(len((bboxes))):
+			st.markdown(f"* Class: {str(category_index[classes[idx]]['name'])}, confidence score: {str(round(scores[idx], 2))}")

internship.md

@@ -0,0 +1,205 @@
+# Luzie internship
+
+* This document contains steps and relevant links to do the steps, but you will still have to read through the links and find your way around the task. If you encounter a problem or an error, search through the documentation or Google it and if you are still stuck after 30 mins, then ask :)
+* At the bottom of each section there are terms that might be new to you. If you don't know them, try to search for their meaning and feel free to write down their definition. There are also some questions included that you can try to answer yourself or we can discuss them at the end of each section
+
+- [Luzie internship](#luzie-internship)
+  - [0. Introduction](#0-introduction)
+  - [1. Create dataset](#1-create-dataset)
+  - [2. Label dataset](#2-label-dataset)
+  - [3. Train model](#3-train-model)
+  - [4. Create website](#4-create-website)
+  - [5. Host it in the cloud](#5-host-it-in-the-cloud)
+  - [6. Improve system](#6-improve-system)
+  - [Resources](#resources)
+
+## 0. Introduction
+
+* The goal of this internship is to gather a dataset of banana images, label them, train a machine learning model, and show the model in a website
+* The Machine Learning goal of this project is to determine whether a banana is **ripe** or **edible**
+* In the end we will have something like: https://huggingface.co/spaces/anebz/test
+
+---
+
+Basic new terms and questions:
+
+* Machine Learning
+* Dataset
+* Model
+* Train a model
+* Classes in machine learning
+* Why only 2 classes? (ripe and edible)
+
+## 1. Create dataset
+
+Create a dataset with images relevant for our project, in this case bananas. Since we want to detect **ripe** and **edible** bananas, we need images of both types. Deep neural networks need thousands of images to learn the patterns in the images, but in this project we will use pre-trained models. With only a few of our images, we can obtain good results. To obtain decent results, we should have at least 50 images of each class.
+
+This section ends when we have a folder of images in our computer.
+
+---
+
+New terms and questions:
+
+* Deep neural networks
+* Pre-trained model
+* Why 50 images and not less?
+
+## 2. Label dataset
+
+Next step is to label images, to mark if the banana is ripe or edible. If we only mark the class in each image, we are doing classification. But in Computer Vision, a popular task is object detection. This means each image needs two markings: a box where the object is, and a label. To practice object detection, we will mark each image with the box and the label.
+
+![ ](https://cv-tricks.com/wp-content/uploads/2021/08/image1.jpeg)
+
+We will use the tool Datature: https://datature.io/
+
+It's a startup that created a web interface to annotate images and train computer vision models. The great part of using this is that we don't need any code at all. Resources:
+
+* Blogpost about labeling and training a face mask detection model: https://datature.io/blog/train-and-visualize-face-mask-detection-model
+* Datature Nexus documentation: https://docs.datature.io/
+
+Use the resources above to find out how to:
+
+* Create Datature Nexus account (free trial)
+* Create a project
+* Upload images
+* Create classes
+* Annotate images
+  * Create rectangular boxes in images
+  * Assign a class to each box
+
+This section is finished when all the images have been annotated and reviewed in Nexus.
+
+---
+
+New terms and questions:
+
+* Computer vision
+* Object detection
+* Bounding box
+* Tag distribution
+* Risks of class imbalance
+
+## 3. Train model
+
+Once we have our images, we can train our model! We will have to create a 'Workflow' in Nexus. Try to use the documentation and blogpost to do the following steps:
+
+* Build training workflow
+  * Select train-test split ratio
+  * Select augmentation
+  * Model settings
+* Train model
+* Monitor model
+  * Loss
+  * Precision and recall
+* Export model
+
+This section is finished when there is a .zip exported in the computer with the trained model parameters.
+
+---
+
+New terms and questions:
+
+* Train-test split ratio
+* Augmentation
+* Checkpoint
+* Model training loss
+  * Should the loss go up/down?
+* What pre-trained model did the workflow use?
+* Precision and recall
+  * Should precision and recall go up/down?
+
+## 4. Create website
+
+We will use Streamlit to create the website: https://streamlit.io/. It uses Python to create a website.
+
+You can find the website application prepared for you in `app.py`. That is the Python file that uses the Streamlit library to do the following steps:
+
+* Load model from the `saved_model/` folder
+* User interface for the user to upload an image
+* Open image, resize, convert to correct format for the model
+* Obtain results from the model prediction, process the data
+* Iterate prediction boxes and plot them on top of the image
+* Show the result image and the prediction boxes
+
+You are not required to write any code, everything is done for you and the code should work. However, please read through the Python file and the comments, and try to have a general understand of what the code does. Check especially the #TODO lines which include parameters that we might have to change depending on our project settings.
+
+If you have Python installed in your computer, you can install Streamlit from the command line:
+
+```bash
+pip install streamlit
+```
+
+Install the libraries that we need to run the Streamlit app. These libraries are in the file `requirements.txt`:
+
+```bash
+pip install -r requirements.txt
+```
+
+And then run the app.py file:
+
+```bash
+cd path/to/project/
+streamlit run app.py
+```
+
+This will open a new window where we can see the website and interact with it: upload images, and check the results.
+
+This section is done when the code is more or less understood, and we have managed to run the app locally and interact with the model in our browser.
+
+---
+
+New terms and questions:
+
+* What does streamlit do?
+* What is pip?
+* What is the confidence threshold?
+
+## 5. Host it in the cloud
+
+For now the script runs in our computer, but we would like to put it in the Internet so anyone can just access a link and get predictions from the model. There are many ways to deploy an app to the cloud, we will use a simple easy way: Huggingface Spaces: https://huggingface.co/spaces/. This is a website from Huggingface where people can showcase their models and interact with them. Sort the spaces by 'Most liked' and explore some of the spaces :)
+
+1. Create Huggingface account
+2. Create a Space
+3. Write a name for the space. Remember, this website will be public, so choose a name that matches what the app does! For example: banana-analysis or something like that
+4. Select Streamlit as the Space SDK
+5. Choose Public
+6. When the space is done, clone the repository in your local computer: git clone ...
+7. Open the folder in Visual Studio code, open README.md and change the emoji for the project
+8. Copy the model files (saved_model/ folder, label_map.pbtxt file) into this folder
+9. Copy app.py and requirements.txt into this folder
+
+Once all the files we need are in the folder, we can push this to the Huggingface Space. Open Git bash and input these commands:
+
+```bash
+git add .
+git commit -m "Added files"
+git push
+```
+
+The app will take some time to upload the files, especially the model files. After the git push is complete, the Space will take some minutes to build the application and show our Streamlit App in the Huggingface Space.
+
+This section is finished when the Streamlit app is displayed in the Space and we can upload an image and obtain predictions.
+
+---
+
+New terms and questions:
+
+* What is Huggingface? What do they do?
+* What is git? Why is it useful?
+  * What is git add, git commit, git push?
+
+## 6. Improve system
+
+We have gathered a dataset, labeled the images, trained a model, exported model parameters, created a Streamlit app and hosted it in Huggingface Spaces. Now everyone can access this link and try their images on our app.
+
+We will review the Space together, try a test dataset and check if the results are satisfactory. Possibly, the dataset will have to be revised, you will need to add more images, label them, train the model again, export the new model's parameters and update these files in the Space. We might have to do this a few times.
+
+In the end, the internship is finished when we have a robust model that can handle all types of images and returns appropriate boxes and labels. Now we can show our app to the world! 🤩
+
+## Resources
+
+* Learn Python
+  * [Text: Python tutorial](https://www.afterhoursprogramming.com/tutorial/python/python-overview/)
+  * [Video: Datacamp course](https://www.datacamp.com/courses/intro-to-python-for-data-science)
+* [Course: Machine Learning](https://www.coursera.org/learn/machine-learning)
+* [Book: Cloud fundamentals](https://cloudresumechallenge.dev/)

label_map.pbtxt

@@ -0,0 +1,10 @@
+item {
+  id: 1
+  name: "ripe"
+}
+
+item {
+  id: 2
+  name: "overripe"
+}
+

packages.txt

@@ -0,0 +1,3 @@
+ffmpeg
+libsm6
+libxext6

pipeline.config

@@ -0,0 +1,127 @@
+model {
+  faster_rcnn {
+    num_classes: 2
+    image_resizer {
+      keep_aspect_ratio_resizer {
+        min_dimension: 800
+        max_dimension: 1333
+        pad_to_max_dimension: true
+      }
+    }
+    feature_extractor {
+      type: "faster_rcnn_inception_resnet_v2_keras"
+    }
+    first_stage_anchor_generator {
+      grid_anchor_generator {
+        height_stride: 16
+        width_stride: 16
+        scales: 0.25
+        scales: 0.5
+        scales: 1.0
+        scales: 2.0
+        aspect_ratios: 0.5
+        aspect_ratios: 1.0
+        aspect_ratios: 2.0
+      }
+    }
+    first_stage_box_predictor_conv_hyperparams {
+      op: CONV
+      regularizer {
+        l2_regularizer {
+          weight: 0.0
+        }
+      }
+      initializer {
+        truncated_normal_initializer {
+          stddev: 0.01
+        }
+      }
+    }
+    first_stage_nms_score_threshold: 0.0
+    first_stage_nms_iou_threshold: 0.7
+    first_stage_max_proposals: 300
+    first_stage_localization_loss_weight: 2.0
+    first_stage_objectness_loss_weight: 1.0
+    initial_crop_size: 17
+    maxpool_kernel_size: 1
+    maxpool_stride: 1
+    second_stage_box_predictor {
+      mask_rcnn_box_predictor {
+        fc_hyperparams {
+          op: FC
+          regularizer {
+            l2_regularizer {
+              weight: 0.0
+            }
+          }
+          initializer {
+            variance_scaling_initializer {
+              factor: 1.0
+              uniform: true
+              mode: FAN_AVG
+            }
+          }
+        }
+        use_dropout: false
+        dropout_keep_probability: 1.0
+      }
+    }
+    second_stage_post_processing {
+      batch_non_max_suppression {
+        score_threshold: 0.0
+        iou_threshold: 0.6
+        max_detections_per_class: 100
+        max_total_detections: 200
+      }
+      score_converter: SOFTMAX
+    }
+    second_stage_localization_loss_weight: 2.0
+    second_stage_classification_loss_weight: 1.0
+  }
+}
+train_config {
+  batch_size: 2
+  optimizer {
+    momentum_optimizer {
+      learning_rate {
+        cosine_decay_learning_rate {
+          learning_rate_base: 0.04
+          total_steps: 1000
+          warmup_learning_rate: 0.0
+          warmup_steps: 250
+        }
+      }
+      momentum_optimizer_value: 0.9
+    }
+    use_moving_average: false
+  }
+  gradient_clipping_by_norm: 10.0
+  fine_tune_checkpoint: "PATH_TO_BE_CONFIGURED"
+  num_steps: 1000
+  max_number_of_boxes: 100
+  fine_tune_checkpoint_type: "detection"
+  use_bfloat16: false
+  fine_tune_checkpoint_version: V2
+}
+train_input_reader {
+  label_map_path: "PATH_TO_BE_CONFIGURED"
+  shuffle: false
+  tf_record_input_reader {
+    input_path: "PATH_TO_BE_CONFIGURED"
+  }
+  num_parallel_batches: 1
+}
+eval_config {
+  metrics_set: "coco_detection_metrics"
+  use_moving_averages: false
+  batch_size: 1
+}
+eval_input_reader {
+  label_map_path: "PATH_TO_BE_CONFIGURED"
+  shuffle: false
+  num_epochs: 1
+  tf_record_input_reader {
+    input_path: "PATH_TO_BE_CONFIGURED"
+  }
+  num_parallel_batches: 1
+}

requirements.txt

@@ -0,0 +1,5 @@
+numpy
+opencv-python-headless
+Pillow
+streamlit
+tensorflow