Paparazzi UAS  v5.14.0_stable-0-g3f680d1
Paparazzi is a free software Unmanned Aircraft System.
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Modules Pages
textons.c
Go to the documentation of this file.
1 /*
2  * Copyright (C) 2016, Hann Woei Ho, Guido de Croon
3  *
4  * This file is part of Paparazzi.
5  *
6  * Paparazzi is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2, or (at your option)
9  * any later version.
10  *
11  * Paparazzi is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with Paparazzi; see the file COPYING. If not, write to
18  * the Free Software Foundation, 59 Temple Place - Suite 330,
19  * Boston, MA 02111-1307, USA.
20  */
21 
32 #include <stdlib.h>
33 #include <stdio.h>
36 
37 float ** **dictionary;
41 
42 // initial settings:
43 #ifndef TEXTONS_LOAD_DICTIONARY
44 #define TEXTONS_LOAD_DICTIONARY 1
45 #endif
46 PRINT_CONFIG_VAR(TEXTONS_LOAD_DICTIONARY)
47 
48 #ifndef TEXTONS_ALPHA
49 #define TEXTONS_ALPHA 10
50 #endif
51 PRINT_CONFIG_VAR(TEXTONS_ALPHA)
52 
53 #ifndef TEXTONS_N_TEXTONS
54 #define TEXTONS_N_TEXTONS 20
55 #endif
56 PRINT_CONFIG_VAR(TEXTONS_N_TEXTONS)
57 
58 #ifndef TEXTONS_N_SAMPLES
59 #define TEXTONS_N_SAMPLES 100
60 #endif
61 PRINT_CONFIG_VAR(TEXTONS_N_SAMPLES)
62 
63 #ifndef TEXTONS_PATCH_SIZE
64 #define TEXTONS_PATCH_SIZE 6
65 #endif
66 PRINT_CONFIG_VAR(TEXTONS_PATCH_SIZE)
67 
68 #ifndef TEXTONS_N_LEARNING_SAMPLES
69 #define TEXTONS_N_LEARNING_SAMPLES 10000
70 #endif
71 PRINT_CONFIG_VAR(TEXTONS_N_LEARNING_SAMPLES)
72 
73 #ifndef TEXTONS_FULL_SAMPLING
74 #define TEXTONS_FULL_SAMPLING 0
75 #endif
76 PRINT_CONFIG_VAR(TEXTONS_FULL_SAMPLING)
77 
78 #ifndef TEXTONS_BORDER_WIDTH
79 #define TEXTONS_BORDER_WIDTH 0
80 #endif
81 PRINT_CONFIG_VAR(TEXTONS_BORDER_WIDTH)
82 
83 #ifndef TEXTONS_BORDER_HEIGHT
84 #define TEXTONS_BORDER_HEIGHT 0
85 #endif
86 PRINT_CONFIG_VAR(TEXTONS_BORDER_HEIGHT)
87 
88 #ifndef TEXTONS_DICTIONARY_NUMBER
89 #define TEXTONS_DICTIONARY_NUMBER 0
90 #endif
91 PRINT_CONFIG_VAR(TEXTONS_DICTIONARY_NUMBER)
92 
93 
104 
105 // status variables
107 float alpha = 0.0;
108 
109 // File pointer for saving the dictionary
110 static FILE *dictionary_logger = NULL;
111 #ifndef DICTIONARY_PATH
112 #define DICTIONARY_PATH /data/video/
113 #endif
114 
120 struct image_t *texton_func(struct image_t *img);
121 struct image_t *texton_func(struct image_t *img)
122 {
123 
124  if (img->buf_size == 0) { return img; }
125 
126  // extract frame from img struct:
127  uint8_t *frame = (uint8_t *)img->buf;
128 
129  // if patch size odd, correct:
130  if (patch_size % 2 == 1) { patch_size++; }
131 
132  // if dictionary not initialized:
133  if (dictionary_ready == 0) {
134  if (load_dictionary == 0) {
135  // Train the dictionary:
136  DictionaryTrainingYUV(frame, img->w, img->h);
137 
138  // After a number of samples, stop learning:
140  // Save the dictionary:
142  // stop learning:
143  dictionary_ready = 1;
144  // lower learning rate
145  alpha = 0.0;
146  }
147  } else {
148  // Load the dictionary:
150  }
151  } else {
152  // Extract distributions
153  DistributionExtraction(frame, img->w, img->h);
154  }
155 
156  return img; // Colorfilter did not make a new image
157 }
158 
166 void DictionaryTrainingYUV(uint8_t *frame, uint16_t width, uint16_t height)
167 {
168  int i, j, w, s, texton, c; // iterators
169  int x, y; // image coordinates
170  float error_texton; // distance between an image patch and a texton
171 
172  uint8_t *buf;
173 
174  // ***********************
175  // DICTIONARY LEARNING
176  // ***********************
177 
178  if (!dictionary_initialized) {
179  // **************
180  // INITIALISATION
181  // **************
182 
183  printf("Intializing dictionary!\n");
184 
185  // in the first image, we initialize the textons to random patches in the image
186  for (w = 0; w < n_textons; w++) {
187  // select a coordinate
188  x = rand() % (width - patch_size);
189  y = rand() % (height - patch_size);
190 
191  //printf("(x,y) = (%d,%d), (w,h) = (%d,%d), ps = %d\n", x, y, width, height, patch_size);
192  // take the sample
193  for (i = 0; i < patch_size; i++) {
194  buf = frame + (width * 2 * (i + y)) + 2 * x;
195  for (j = 0; j < patch_size; j++) {
196  // put it in a texton
197  printf("Setting dictionary:\n");
198  // U/V component
199 
200  dictionary[w][i][j][0] = (float) * buf;
201  buf += 1;
202  // Y1/Y2 component
203  dictionary[w][i][j][1] = (float) * buf;
204  buf += 1;
205  printf("Done!\n");
206  }
207  }
208  }
210  } else {
211  // ********
212  // LEARNING
213  // ********
214  printf("Learning!");
215  alpha = ((float) alpha_uint) / 255.0;
216 
217  float *texton_distances, * **patch;
218  texton_distances = (float *)calloc(n_textons, sizeof(float));
219  patch = (float ** *)calloc(patch_size, sizeof(float **));
220 
221  for (i = 0; i < patch_size; i++) {
222  patch[i] = (float **)calloc(patch_size, sizeof(float *));
223  for (j = 0; j < patch_size; j++) {
224  patch[i][j] = (float *)calloc(2, sizeof(float));
225  }
226  }
227 
228  // Extract and learn from n_samples_image per image
229  for (s = 0; s < n_samples_image; s++) {
230  // select a random sample from the image
231  x = rand() % (width - patch_size);
232  y = rand() % (height - patch_size);
233 
234  // reset texton_distances
235  for (texton = 0; texton < n_textons; texton++) {
236  texton_distances[texton] = 0;
237  }
238 
239  // extract sample
240  for (i = 0; i < patch_size; i++) {
241  buf = frame + (width * 2 * (i + y)) + 2 * x;
242  for (j = 0; j < patch_size; j++) {
243  // U/V component
244  patch[i][j][0] = (float) * buf;
245  buf += 1;
246  // Y1/Y2 component
247  patch[i][j][1] = (float) * buf;
248  buf += 1;
249  }
250  }
251 
252  // determine distances to the textons:
253  for (i = 0; i < patch_size; i++) {
254  for (j = 0; j < patch_size; j++) {
255  for (c = 0; c < 2; c++) {
256  // determine the distance to textons
257  for (texton = 0; texton < n_textons; texton++) {
258  texton_distances[texton] += (patch[i][j][c] - dictionary[texton][i][j][c])
259  * (patch[i][j][c] - dictionary[texton][i][j][c]);
260  }
261  }
262  }
263  }
264 
265  // search the closest texton
266  int assignment = 0;
267  float min_dist = texton_distances[0];
268  for (texton = 1; texton < n_textons; texton++) {
269  if (texton_distances[texton] < min_dist) {
270  min_dist = texton_distances[texton];
271  assignment = texton;
272  }
273  }
274 
275  // move the neighbour closer to the input
276  for (i = 0; i < patch_size; i++) {
277  for (j = 0; j < patch_size; j++) {
278  for (c = 0; c < 2; c++) {
279  error_texton = patch[i][j][c] - dictionary[assignment][i][j][c];
280  dictionary[assignment][i][j][c] += (alpha * error_texton);
281  }
282  }
283  }
284 
285  // Augment the number of learned samples:
286  learned_samples++;
287  }
288 
289  // Free the allocated memory:
290  for (i = 0; i < patch_size; i++) {
291  for (j = 0; j < patch_size; j++) {
292  free(patch[i][j]);
293  }
294  free(patch[i]);
295  }
296  free(patch);
297  free(texton_distances);
298  }
299 
300  // Free the buffer
301  buf = NULL;
302  free(buf);
303 }
304 
311 void DistributionExtraction(uint8_t *frame, uint16_t width, uint16_t height)
312 {
313  int i, j, texton, c; // iterators
314  int x, y; // coordinates
315  int n_extracted_textons = 0;
316 
317  uint8_t *buf;
318 
319  // ************************
320  // EXECUTION
321  // ************************
322 
323  printf("Execute!\n");
324 
325  // Allocate memory for texton distances and image patch:
326  float *texton_distances, * **patch;
327  texton_distances = (float *)calloc(n_textons, sizeof(float));
328  patch = (float ** *)calloc(patch_size, sizeof(float **));
329  for (i = 0; i < patch_size; i++) {
330  patch[i] = (float **)calloc(patch_size, sizeof(float *));
331  for (j = 0; j < patch_size; j++) {
332  patch[i][j] = (float *)calloc(2, sizeof(float));
333  }
334  }
335 
336  int finished = 0;
337  x = 0;
338  y = 0;
339  while (!finished) {
340  if (!FULL_SAMPLING) {
341  x = border_width + rand() % (width - patch_size - 2 * border_width);
342  y = border_height + rand() % (height - patch_size - 2 * border_height);
343  }
344 
345  // reset texton_distances
346  for (texton = 0; texton < n_textons; texton++) {
347  texton_distances[texton] = 0;
348  }
349 
350  // extract sample
351  for (i = 0; i < patch_size; i++) {
352  buf = frame + (width * 2 * (i + y)) + 2 * x;
353  for (j = 0; j < patch_size; j++) {
354  // U/V component
355  patch[i][j][0] = (float) * buf;
356  buf += 1;
357  // Y1/Y2 component
358  patch[i][j][1] = (float) * buf;
359  buf += 1;
360  }
361  }
362 
363  // determine distances:
364  for (i = 0; i < patch_size; i++) {
365  for (j = 0; j < patch_size; j++) {
366  for (c = 0; c < 2; c++) {
367  // determine the distance to words
368  for (texton = 0; texton < n_textons; texton++) {
369  texton_distances[texton] += (patch[i][j][c] - dictionary[texton][i][j][c])
370  * (patch[i][j][c] - dictionary[texton][i][j][c]);
371  }
372  }
373  }
374  }
375 
376  // determine the nearest neighbour
377  // search the closest centroid
378  int assignment = 0;
379  float min_dist = texton_distances[0];
380  for (texton = 1; texton < n_textons; texton++) {
381  if (texton_distances[texton] < min_dist) {
382  min_dist = texton_distances[texton];
383  assignment = texton;
384  }
385  }
386 
387  // put the assignment in the histogram
388  texton_distribution[assignment]++;
389  n_extracted_textons++;
390 
391  if (!FULL_SAMPLING && n_extracted_textons == n_samples_image) {
392  finished = 1;
393  } else {
394  // FULL_SAMPLING is actually a sampling that covers the image:
395  y += patch_size;
396  // True full sampling would require:
397  // y++;
398 
399  if (y > height - patch_size) {
400  if (!FULL_SAMPLING) {
401  x += patch_size;
402  } else {
403  x++;
404  }
405  y = 0;
406  }
407  if (x > width - patch_size) {
408  finished = 1;
409  }
410  }
411  }
412 
413  // Normalize distribution:
414  for (i = 0; i < n_textons; i++) {
415  texton_distribution[i] = texton_distribution[i] / (float) n_extracted_textons;
416  // printf("textons[%d] = %f\n", i, texton_distribution[i]);
417  }
418  // printf("\n");
419 
420 
421  // free memory:
422  for (i = 0; i < patch_size; i++) {
423  for (j = 0; j < patch_size; j++) {
424  free(patch[i][j]);
425  }
426  free(patch[i]);
427  }
428  free(patch);
429  free(texton_distances);
430 
431  buf = NULL;
432  free(buf);
433 
434 } // EXECUTION
435 
436 
437 
442 {
443  //save a dictionary
444  char filename[512];
445 
446  // Check for available files
447  sprintf(filename, "%s/Dictionary_%05d.dat", STRINGIFY(DICTIONARY_PATH), dictionary_number);
448 
449  dictionary_logger = fopen(filename, "w");
450 
451  if (dictionary_logger == NULL) {
452  perror("Error while opening the file.\n");
453  } else {
454  // (over-)write dictionary
455  for (uint8_t i = 0; i < n_textons; i++) {
456  for (uint8_t j = 0; j < patch_size; j++) {
457  for (uint8_t k = 0; k < patch_size; k++) {
458  fprintf(dictionary_logger, "%f\n", dictionary[i][j][k][0]);
459  fprintf(dictionary_logger, "%f\n", dictionary[i][j][k][1]);
460  }
461  }
462  }
463  fclose(dictionary_logger);
464  }
465 
466 }
467 
472 {
473  char filename[512];
474  sprintf(filename, "%s/Dictionary_%05d.dat", STRINGIFY(DICTIONARY_PATH), dictionary_number);
475 
476  if ((dictionary_logger = fopen(filename, "r"))) {
477  // Load the dictionary:
478  for (int i = 0; i < n_textons; i++) {
479  for (int j = 0; j < patch_size; j++) {
480  for (int k = 0; k < patch_size; k++) {
481  if (fscanf(dictionary_logger, "%f\n", &dictionary[i][j][k][0]) == EOF) { break; }
482  if (fscanf(dictionary_logger, "%f\n", &dictionary[i][j][k][1]) == EOF) { break; }
483  }
484  }
485  }
486 
487  fclose(dictionary_logger);
488  dictionary_ready = 1;
489  } else {
490  // If the given dictionary does not exist, we start learning one:
491  printf("Texton dictionary %d does not exist, we start learning one.\n", dictionary_number);
492  load_dictionary = 0;
493  learned_samples = 0;
495  }
496 }
497 
501 void textons_init(void)
502 {
503  printf("Textons init\n");
504  texton_distribution = (float *)calloc(n_textons, sizeof(float));
506  learned_samples = 0;
507  dictionary_ready = 0;
508  dictionary = (float ** **)calloc(n_textons, sizeof(float ** *));
509  for (int w = 0; w < n_textons; w++) {
510  dictionary[w] = (float ** *) calloc(patch_size, sizeof(float **));
511  for (int i = 0; i < patch_size; i++) {
512  dictionary[w][i] = (float **) calloc(patch_size, sizeof(float *));
513  for (int j = 0; j < patch_size; j++) {
514  dictionary[w][i][j] = (float *) calloc(2, sizeof(float));
515  }
516  }
517  }
518 
519  cv_add(texton_func);
520 }
521 
522 void textons_stop(void)
523 {
524  free(texton_distribution);
525  free(dictionary);
526 }
527 
uint32_t n_samples_image
Definition: textons.c:99
unsigned short uint16_t
Definition: types.h:16
uint8_t load_dictionary
Definition: textons.c:94
#define TEXTONS_PATCH_SIZE
Definition: textons.c:64
uint32_t buf_size
The buffer size.
Definition: image.h:52
uint32_t border_height
Definition: textons.c:102
#define TEXTONS_FULL_SAMPLING
Definition: textons.c:74
float alpha
Definition: textons.c:107
Definition: image.h:43
uint32_t border_width
Definition: textons.c:101
uint8_t patch_size
Definition: textons.c:97
uint8_t dictionary_number
Definition: textons.c:103
Takes an image and represents the texture and colors in the image with a texton histogram.
void DistributionExtraction(uint8_t *frame, uint16_t width, uint16_t height)
Function that extracts a texton histogram from an image.
Definition: textons.c:311
#define EOF
Definition: usb_ser_hw.c:87
#define TEXTONS_DICTIONARY_NUMBER
Definition: textons.c:89
void save_texton_dictionary(void)
Save the texton dictionary.
Definition: textons.c:441
void textons_init(void)
Initialize.
Definition: textons.c:501
#define TEXTONS_LOAD_DICTIONARY
Definition: textons.c:44
uint32_t n_learning_samples
Definition: textons.c:98
uint8_t alpha_uint
Definition: textons.c:95
#define TEXTONS_ALPHA
Definition: textons.c:49
void load_texton_dictionary(void)
Load a texton dictionary.
Definition: textons.c:471
uint16_t w
Image width.
Definition: image.h:45
unsigned long uint32_t
Definition: types.h:18
Computer vision framework for onboard processing.
uint16_t h
Image height.
Definition: image.h:46
void textons_stop(void)
Definition: textons.c:522
void * buf
Image buffer (depending on the image_type)
Definition: image.h:53
uint8_t n_textons
Definition: textons.c:96
#define TEXTONS_N_TEXTONS
Definition: textons.c:54
#define TEXTONS_N_LEARNING_SAMPLES
Definition: textons.c:69
unsigned char uint8_t
Definition: types.h:14
static FILE * dictionary_logger
Definition: textons.c:110
float **** dictionary
Definition: textons.c:37
uint8_t dictionary_ready
Definition: textons.c:106
struct image_t * texton_func(struct image_t *img)
Main texton processing function that first either loads or learns a dictionary and then extracts the ...
Definition: textons.c:121
uint8_t dictionary_initialized
Definition: textons.c:39
uint32_t learned_samples
Definition: textons.c:38
uint8_t FULL_SAMPLING
Definition: textons.c:100
#define TEXTONS_N_SAMPLES
Definition: textons.c:59
#define DICTIONARY_PATH
Definition: textons.c:112
#define TEXTONS_BORDER_WIDTH
Definition: textons.c:79
float * texton_distribution
Definition: textons.c:40
void DictionaryTrainingYUV(uint8_t *frame, uint16_t width, uint16_t height)
Function that performs one pass for dictionary training.
Definition: textons.c:166
#define TEXTONS_BORDER_HEIGHT
Definition: textons.c:84