kumar989 pritish commited on
Commit
5edb1fc
0 Parent(s):

Duplicate from pritish/Image-Captioning

Browse files

Co-authored-by: Pritish Mishra <pritish@users.noreply.huggingface.co>

.gitattributes ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ftz filter=lfs diff=lfs merge=lfs -text
6
+ *.gz filter=lfs diff=lfs merge=lfs -text
7
+ *.h5 filter=lfs diff=lfs merge=lfs -text
8
+ *.joblib filter=lfs diff=lfs merge=lfs -text
9
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
10
+ *.model filter=lfs diff=lfs merge=lfs -text
11
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
12
+ *.npy filter=lfs diff=lfs merge=lfs -text
13
+ *.npz filter=lfs diff=lfs merge=lfs -text
14
+ *.onnx filter=lfs diff=lfs merge=lfs -text
15
+ *.ot filter=lfs diff=lfs merge=lfs -text
16
+ *.parquet filter=lfs diff=lfs merge=lfs -text
17
+ *.pickle filter=lfs diff=lfs merge=lfs -text
18
+ *.pkl filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pt filter=lfs diff=lfs merge=lfs -text
21
+ *.pth filter=lfs diff=lfs merge=lfs -text
22
+ *.rar filter=lfs diff=lfs merge=lfs -text
23
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
24
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
25
+ *.tflite filter=lfs diff=lfs merge=lfs -text
26
+ *.tgz filter=lfs diff=lfs merge=lfs -text
27
+ *.wasm filter=lfs diff=lfs merge=lfs -text
28
+ *.xz filter=lfs diff=lfs merge=lfs -text
29
+ *.zip filter=lfs diff=lfs merge=lfs -text
30
+ *.zst filter=lfs diff=lfs merge=lfs -text
31
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
32
+ *.file filter=lfs diff=lfs merge=lfs -text
33
+ vocab_2.file filter=lfs diff=lfs merge=lfs -text
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ __pycache__/*
README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Image Captioning
3
+ emoji: 🦀
4
+ colorFrom: red
5
+ colorTo: gray
6
+ sdk: streamlit
7
+ sdk_version: 1.10.0
8
+ app_file: app.py
9
+ pinned: false
10
+ duplicated_from: pritish/Image-Captioning
11
+ ---
12
+
13
+ Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import io
2
+ import os
3
+ import streamlit as st
4
+ import requests
5
+ from PIL import Image
6
+ from model import get_caption_model, generate_caption
7
+
8
+
9
+ @st.cache(allow_output_mutation=True)
10
+ def get_model():
11
+ return get_caption_model()
12
+
13
+ caption_model = get_model()
14
+
15
+
16
+ def predict():
17
+ captions = []
18
+ pred_caption = generate_caption('tmp.jpg', caption_model)
19
+
20
+ st.markdown('#### Predicted Captions:')
21
+ captions.append(pred_caption)
22
+
23
+ for _ in range(4):
24
+ pred_caption = generate_caption('tmp.jpg', caption_model, add_noise=True)
25
+ if pred_caption not in captions:
26
+ captions.append(pred_caption)
27
+
28
+ for c in captions:
29
+ st.write(c)
30
+
31
+ st.title('Image Captioner')
32
+ img_url = st.text_input(label='Enter Image URL')
33
+
34
+ if (img_url != "") and (img_url != None):
35
+ img = Image.open(requests.get(img_url, stream=True).raw)
36
+ img = img.convert('RGB')
37
+ st.image(img)
38
+ img.save('tmp.jpg')
39
+ predict()
40
+ os.remove('tmp.jpg')
41
+
42
+
43
+ st.markdown('<center style="opacity: 70%">OR</center>', unsafe_allow_html=True)
44
+ img_upload = st.file_uploader(label='Upload Image', type=['jpg', 'png', 'jpeg'])
45
+
46
+ if img_upload != None:
47
+ img = img_upload.read()
48
+ img = Image.open(io.BytesIO(img))
49
+ img = img.convert('RGB')
50
+ img.save('tmp.jpg')
51
+ st.image(img)
52
+ predict()
53
+ os.remove('tmp.jpg')
model.py ADDED
@@ -0,0 +1,328 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pickle
2
+ import tensorflow as tf
3
+ import pandas as pd
4
+ import numpy as np
5
+
6
+
7
+ # CONTANTS
8
+ MAX_LENGTH = 40
9
+ # VOCABULARY_SIZE = 10000
10
+ BATCH_SIZE = 32
11
+ BUFFER_SIZE = 1000
12
+ EMBEDDING_DIM = 512
13
+ UNITS = 512
14
+
15
+
16
+ # LOADING DATA
17
+ vocab = pickle.load(open('saved_vocabulary/vocab_coco.file', 'rb'))
18
+
19
+ tokenizer = tf.keras.layers.TextVectorization(
20
+ # max_tokens=VOCABULARY_SIZE,
21
+ standardize=None,
22
+ output_sequence_length=MAX_LENGTH,
23
+ vocabulary=vocab
24
+ )
25
+
26
+ idx2word = tf.keras.layers.StringLookup(
27
+ mask_token="",
28
+ vocabulary=tokenizer.get_vocabulary(),
29
+ invert=True
30
+ )
31
+
32
+
33
+ # MODEL
34
+ def CNN_Encoder():
35
+ inception_v3 = tf.keras.applications.InceptionV3(
36
+ include_top=False,
37
+ weights='imagenet'
38
+ )
39
+
40
+ output = inception_v3.output
41
+ output = tf.keras.layers.Reshape(
42
+ (-1, output.shape[-1]))(output)
43
+
44
+ cnn_model = tf.keras.models.Model(inception_v3.input, output)
45
+ return cnn_model
46
+
47
+
48
+ class TransformerEncoderLayer(tf.keras.layers.Layer):
49
+
50
+ def __init__(self, embed_dim, num_heads):
51
+ super().__init__()
52
+ self.layer_norm_1 = tf.keras.layers.LayerNormalization()
53
+ self.layer_norm_2 = tf.keras.layers.LayerNormalization()
54
+ self.attention = tf.keras.layers.MultiHeadAttention(
55
+ num_heads=num_heads, key_dim=embed_dim)
56
+ self.dense = tf.keras.layers.Dense(embed_dim, activation="relu")
57
+
58
+
59
+ def call(self, x, training):
60
+ x = self.layer_norm_1(x)
61
+ x = self.dense(x)
62
+
63
+ attn_output = self.attention(
64
+ query=x,
65
+ value=x,
66
+ key=x,
67
+ attention_mask=None,
68
+ training=training
69
+ )
70
+
71
+ x = self.layer_norm_2(x + attn_output)
72
+ return x
73
+
74
+
75
+ class Embeddings(tf.keras.layers.Layer):
76
+
77
+ def __init__(self, vocab_size, embed_dim, max_len):
78
+ super().__init__()
79
+ self.token_embeddings = tf.keras.layers.Embedding(
80
+ vocab_size, embed_dim)
81
+ self.position_embeddings = tf.keras.layers.Embedding(
82
+ max_len, embed_dim, input_shape=(None, max_len))
83
+
84
+
85
+ def call(self, input_ids):
86
+ length = tf.shape(input_ids)[-1]
87
+ position_ids = tf.range(start=0, limit=length, delta=1)
88
+ position_ids = tf.expand_dims(position_ids, axis=0)
89
+
90
+ token_embeddings = self.token_embeddings(input_ids)
91
+ position_embeddings = self.position_embeddings(position_ids)
92
+
93
+ return token_embeddings + position_embeddings
94
+
95
+
96
+ class TransformerDecoderLayer(tf.keras.layers.Layer):
97
+
98
+ def __init__(self, embed_dim, units, num_heads):
99
+ super().__init__()
100
+ self.embedding = Embeddings(
101
+ tokenizer.vocabulary_size(), embed_dim, MAX_LENGTH)
102
+
103
+ self.attention_1 = tf.keras.layers.MultiHeadAttention(
104
+ num_heads=num_heads, key_dim=embed_dim, dropout=0.1
105
+ )
106
+ self.attention_2 = tf.keras.layers.MultiHeadAttention(
107
+ num_heads=num_heads, key_dim=embed_dim, dropout=0.1
108
+ )
109
+
110
+ self.layernorm_1 = tf.keras.layers.LayerNormalization()
111
+ self.layernorm_2 = tf.keras.layers.LayerNormalization()
112
+ self.layernorm_3 = tf.keras.layers.LayerNormalization()
113
+
114
+ self.ffn_layer_1 = tf.keras.layers.Dense(units, activation="relu")
115
+ self.ffn_layer_2 = tf.keras.layers.Dense(embed_dim)
116
+
117
+ self.out = tf.keras.layers.Dense(tokenizer.vocabulary_size(), activation="softmax")
118
+
119
+ self.dropout_1 = tf.keras.layers.Dropout(0.3)
120
+ self.dropout_2 = tf.keras.layers.Dropout(0.5)
121
+
122
+
123
+ def call(self, input_ids, encoder_output, training, mask=None):
124
+ embeddings = self.embedding(input_ids)
125
+
126
+ combined_mask = None
127
+ padding_mask = None
128
+
129
+ if mask is not None:
130
+ causal_mask = self.get_causal_attention_mask(embeddings)
131
+ padding_mask = tf.cast(mask[:, :, tf.newaxis], dtype=tf.int32)
132
+ combined_mask = tf.cast(mask[:, tf.newaxis, :], dtype=tf.int32)
133
+ combined_mask = tf.minimum(combined_mask, causal_mask)
134
+
135
+ attn_output_1 = self.attention_1(
136
+ query=embeddings,
137
+ value=embeddings,
138
+ key=embeddings,
139
+ attention_mask=combined_mask,
140
+ training=training
141
+ )
142
+
143
+ out_1 = self.layernorm_1(embeddings + attn_output_1)
144
+
145
+ attn_output_2 = self.attention_2(
146
+ query=out_1,
147
+ value=encoder_output,
148
+ key=encoder_output,
149
+ attention_mask=padding_mask,
150
+ training=training
151
+ )
152
+
153
+ out_2 = self.layernorm_2(out_1 + attn_output_2)
154
+
155
+ ffn_out = self.ffn_layer_1(out_2)
156
+ ffn_out = self.dropout_1(ffn_out, training=training)
157
+ ffn_out = self.ffn_layer_2(ffn_out)
158
+
159
+ ffn_out = self.layernorm_3(ffn_out + out_2)
160
+ ffn_out = self.dropout_2(ffn_out, training=training)
161
+ preds = self.out(ffn_out)
162
+ return preds
163
+
164
+
165
+ def get_causal_attention_mask(self, inputs):
166
+ input_shape = tf.shape(inputs)
167
+ batch_size, sequence_length = input_shape[0], input_shape[1]
168
+ i = tf.range(sequence_length)[:, tf.newaxis]
169
+ j = tf.range(sequence_length)
170
+ mask = tf.cast(i >= j, dtype="int32")
171
+ mask = tf.reshape(mask, (1, input_shape[1], input_shape[1]))
172
+ mult = tf.concat(
173
+ [tf.expand_dims(batch_size, -1), tf.constant([1, 1], dtype=tf.int32)],
174
+ axis=0
175
+ )
176
+ return tf.tile(mask, mult)
177
+
178
+
179
+ class ImageCaptioningModel(tf.keras.Model):
180
+
181
+ def __init__(self, cnn_model, encoder, decoder, image_aug=None):
182
+ super().__init__()
183
+ self.cnn_model = cnn_model
184
+ self.encoder = encoder
185
+ self.decoder = decoder
186
+ self.image_aug = image_aug
187
+ self.loss_tracker = tf.keras.metrics.Mean(name="loss")
188
+ self.acc_tracker = tf.keras.metrics.Mean(name="accuracy")
189
+
190
+
191
+ def calculate_loss(self, y_true, y_pred, mask):
192
+ loss = self.loss(y_true, y_pred)
193
+ mask = tf.cast(mask, dtype=loss.dtype)
194
+ loss *= mask
195
+ return tf.reduce_sum(loss) / tf.reduce_sum(mask)
196
+
197
+
198
+ def calculate_accuracy(self, y_true, y_pred, mask):
199
+ accuracy = tf.equal(y_true, tf.argmax(y_pred, axis=2))
200
+ accuracy = tf.math.logical_and(mask, accuracy)
201
+ accuracy = tf.cast(accuracy, dtype=tf.float32)
202
+ mask = tf.cast(mask, dtype=tf.float32)
203
+ return tf.reduce_sum(accuracy) / tf.reduce_sum(mask)
204
+
205
+
206
+ def compute_loss_and_acc(self, img_embed, captions, training=True):
207
+ encoder_output = self.encoder(img_embed, training=True)
208
+ y_input = captions[:, :-1]
209
+ y_true = captions[:, 1:]
210
+ mask = (y_true != 0)
211
+ y_pred = self.decoder(
212
+ y_input, encoder_output, training=True, mask=mask
213
+ )
214
+ loss = self.calculate_loss(y_true, y_pred, mask)
215
+ acc = self.calculate_accuracy(y_true, y_pred, mask)
216
+ return loss, acc
217
+
218
+
219
+ def train_step(self, batch):
220
+ imgs, captions = batch
221
+
222
+ if self.image_aug:
223
+ imgs = self.image_aug(imgs)
224
+
225
+ img_embed = self.cnn_model(imgs)
226
+
227
+ with tf.GradientTape() as tape:
228
+ loss, acc = self.compute_loss_and_acc(
229
+ img_embed, captions
230
+ )
231
+
232
+ train_vars = (
233
+ self.encoder.trainable_variables + self.decoder.trainable_variables
234
+ )
235
+ grads = tape.gradient(loss, train_vars)
236
+ self.optimizer.apply_gradients(zip(grads, train_vars))
237
+ self.loss_tracker.update_state(loss)
238
+ self.acc_tracker.update_state(acc)
239
+
240
+ return {"loss": self.loss_tracker.result(), "acc": self.acc_tracker.result()}
241
+
242
+
243
+ def test_step(self, batch):
244
+ imgs, captions = batch
245
+
246
+ img_embed = self.cnn_model(imgs)
247
+
248
+ loss, acc = self.compute_loss_and_acc(
249
+ img_embed, captions, training=False
250
+ )
251
+
252
+ self.loss_tracker.update_state(loss)
253
+ self.acc_tracker.update_state(acc)
254
+
255
+ return {"loss": self.loss_tracker.result(), "acc": self.acc_tracker.result()}
256
+
257
+ @property
258
+ def metrics(self):
259
+ return [self.loss_tracker, self.acc_tracker]
260
+
261
+
262
+ def load_image_from_path(img_path):
263
+ img = tf.io.read_file(img_path)
264
+ img = tf.io.decode_jpeg(img, channels=3)
265
+ img = tf.keras.layers.Resizing(299, 299)(img)
266
+ img = tf.keras.applications.inception_v3.preprocess_input(img)
267
+ return img
268
+
269
+
270
+ def generate_caption(img, caption_model, add_noise=False):
271
+ if isinstance(img, str):
272
+ img = load_image_from_path(img)
273
+
274
+ if add_noise == True:
275
+ noise = tf.random.normal(img.shape)*0.1
276
+ img = (img + noise)
277
+ img = (img - tf.reduce_min(img))/(tf.reduce_max(img) - tf.reduce_min(img))
278
+
279
+ img = tf.expand_dims(img, axis=0)
280
+ img_embed = caption_model.cnn_model(img)
281
+ img_encoded = caption_model.encoder(img_embed, training=False)
282
+
283
+ y_inp = '[start]'
284
+ for i in range(MAX_LENGTH-1):
285
+ tokenized = tokenizer([y_inp])[:, :-1]
286
+ mask = tf.cast(tokenized != 0, tf.int32)
287
+ pred = caption_model.decoder(
288
+ tokenized, img_encoded, training=False, mask=mask)
289
+
290
+ pred_idx = np.argmax(pred[0, i, :])
291
+ pred_word = idx2word(pred_idx).numpy().decode('utf-8')
292
+ if pred_word == '[end]':
293
+ break
294
+
295
+ y_inp += ' ' + pred_word
296
+
297
+ y_inp = y_inp.replace('[start] ', '')
298
+ return y_inp
299
+
300
+
301
+ def get_caption_model():
302
+ encoder = TransformerEncoderLayer(EMBEDDING_DIM, 1)
303
+ decoder = TransformerDecoderLayer(EMBEDDING_DIM, UNITS, 8)
304
+
305
+ cnn_model = CNN_Encoder()
306
+
307
+ caption_model = ImageCaptioningModel(
308
+ cnn_model=cnn_model, encoder=encoder, decoder=decoder, image_aug=None,
309
+ )
310
+
311
+ def call_fn(batch, training):
312
+ return batch
313
+
314
+ caption_model.call = call_fn
315
+ sample_x, sample_y = tf.random.normal((1, 299, 299, 3)), tf.zeros((1, 40))
316
+
317
+ caption_model((sample_x, sample_y))
318
+
319
+ sample_img_embed = caption_model.cnn_model(sample_x)
320
+ sample_enc_out = caption_model.encoder(sample_img_embed, training=False)
321
+ caption_model.decoder(sample_y, sample_enc_out, training=False)
322
+
323
+ try:
324
+ caption_model.load_weights('saved_models/image_captioning_coco_weights.h5')
325
+ except FileNotFoundError:
326
+ caption_model.load_weights('Image-Captioning/saved_models/image_captioning_coco_weights.h5')
327
+
328
+ return caption_model
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ numpy==1.22.3
2
+ pandas==1.4.3
3
+ pandas_stubs==1.2.0.56
4
+ Pillow==9.2.0
5
+ requests==2.27.1
6
+ streamlit==1.11.1
7
+ tensorflow==2.9.1
saved_models/image_captioning_coco_weights.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1e84cba7f9b81e113d6cf938f1c03eff9be2f91932edf36ba1d43a14a63eeb23
3
+ size 214197308
saved_models/image_captioning_transformer_weights_2.h5 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e927884d1ad5adc141cdfffb429ba2aba1ad0a1e42e7d9d999972eaf3e5e81e8
3
+ size 201651096
saved_vocabulary/vocab_1.file ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4df22d5dd84bbb3421dfce39b4affd49ba2dc08739627ee9cf036ae17d326d28
3
+ size 860291
saved_vocabulary/vocab_coco.file ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41e68329402249568d75664e814571ccfb8471da72894cffca32cb534a9ef1ea
3
+ size 1153949
test.ipynb ADDED
The diff for this file is too large to render. See raw diff