make-unicode-poster

#!/usr/bin/env jruby

require 'tileinfo.rb'
require 'thread'
require 'tempfile'

include Java

def jruby_safe_backticks(command,output_expected=false)
  # This is horrible, but for some reason IO.popen (like backticks)
  # sometimes still fails with an IOError, so retry up to three times.
  # See my bug report here:
  #   http://jira.codehaus.org/browse/JRUBY-4909
  attempts = 0
  while attempts < 3
    begin
      io = IO.popen(command)
      data = io.read
      io.close
      unless output_expected and (data.length == 0)
        return data
      end
    rescue IOError => e
      STDERR.puts "IOError (#{e.to_s}) running #{command}"
    end
    attempts += 1
  end
  STDERR.puts "Still failed after #{attempts} attempts: #{command}"
  return ""
end

# The directory with the helper files:
$helpers = "png-find-grid-revised"

unless system("cd #{$helpers} && make")
  puts "Failed to build the helper programs.  Is libpng12-dev installed?"
  exit(-1)
end

pdf_name = 'CodeCharts.pdf'
pdf_url = "http://www.unicode.org/Public/6.0.0/charts/CodeCharts.pdf"

last_png_file = 'U-849.png'

unless FileTest.exist?(pdf_name)
  system("curl","-q","-o",pdf_name,pdf_url)
end

unless FileTest.exist?(last_png_file)
  unless system( "pdftoppm",
                 "-r", "800",
                 "-png",
                 pdf_name,
                 "U")
    puts "Failed to convert #{pdf_name} to a series of PNG files"
    exit(-1)
  end
end

puts "Globbing directory..."

png_compare = lambda { |a| a =~ /U-([0-9]+)\.png/; $1.to_i(10) }

png_files = Dir['U-*.png']
png_files = png_files.sort_by(&png_compare)

$pngs_empty = {}

Dir['U-*.png.empty'].each do |already_done|
  $pngs_empty[already_done] = true
end

def find_grid_and_crop(png_file)

  png_output_template = "individual-characters/" + png_file.gsub(/\.png/,"-%08d.png")

  skipping = false

  $mutex.synchronize {
    $condition.wait($mutex) while $running_threads >= $max_threads
    $running_threads += 1
    $condition.signal if $running_threads < $max_threads

    if $pngs_empty[png_file+".empty"]
      puts "Looks as if we've previously found #{png_file} to be empty; skipping..."
      skipping = true
    elsif FileTest.exist? sprintf(png_output_template,0)
      puts "At least one output file from "+png_file+"; skipping..."
      skipping = true
    else
      puts "Splitting: "+png_file
    end
  }

  unless skipping

    last_in_each_cell_y = nil
    first_in_each_cell_y = nil

    last_in_each_cell_x = nil
    first_in_each_cell_x = nil

    # The find-grid program gets certain code pages wrong, and one of
    # the PDF files shouldn't be in there.

    #      if png_file =~ /U2580-002\.png/
    #
    #        # Fix this one...
    #
    #        last_in_each_cell_x  = [ 2247, 2592 ]
    #        last_in_each_cell_y  = [ 1244, 1684,   2124,   2564,   3004,   3444,   3884,   4324, 4764,   5204,   5644,   6084,   6524,   6964,   7404,   7837 ]
    #        first_in_each_cell_x = [ 1908, 2252 ]
    #        first_in_each_cell_y = [ 816,  1249,   1689,   2129,   2569,   3009,   3449,   3889, 4329,   4769,   5209,   5649,   6089,   6529,   6969,   7409 ]
    #
    #      elsif png_file =~ /UFFF0-002\.png/
    #
    #        last_in_each_cell_x = [ 2414 ]
    #        last_in_each_cell_y = [ 5204,   5644,   6084,   6524,   6964 ]
    #        first_in_each_cell_x = [ 2085 ]
    #        first_in_each_cell_y = [ 4769,   5209,   5649,   6089,   6529 ]
    #
    #      elsif png_file =~ /UFB50-004\.png/
    #
    #        last_in_each_cell_x  = [ 1387,   1739,   2090,   2442,   2794,   3146,   3497,   3849,   4200,   4552,   4904,   5952 ]
    #        last_in_each_cell_y  = [ 1244,   1684,   2124,   2564,   3004,   3444,   3884,   4324,   4764,   5204,   5644,   6084,   6524,   6964,   7404,   7837 ]
    #        first_in_each_cell_x = [ 1041,   1392,   1744,   2095,   2447,   2799,   3151,   3502,   3854,   4205,   4557,   5612 ]
    #        first_in_each_cell_y = [ 816,    1249,   1689,   2129,   2569,   3009,   3449,   3889,   4329,   4769,   5209,   5649,   6089,   6529,   6969,   7409 ]

    if false

    else

      command = "#{$helpers}/find-grid #{png_file} 333 427"
      grid_results_output = jruby_safe_backticks(command,output_expected=true)
      if grid_results_output.length > 1
        grid_results_lines = grid_results_output.split("\n")
        grid_results_lines.each do |line|
          line.chomp!
          next if line =~ /image height is/
          values = line.gsub(/^(.*): *()/,'\2').split(/ +/)
          name = line.gsub(/^ *(.*):.*$/,'\1')
          case name
          when 'last_in_each_cell_x'
            last_in_each_cell_x = values
          when 'last_in_each_cell_y'
            last_in_each_cell_y = values
          when 'first_in_each_cell_x'
            first_in_each_cell_x = values
          when 'first_in_each_cell_y'
            first_in_each_cell_y = values
          else
            STDERR.puts "Unparseable line from command '#{command}': #{line}"
            exit(-1)
          end
        end
      else
        STDERR.puts "'#{command}' failed"
        exit(-1)
      end

    end

    cells_width = last_in_each_cell_x.length
    cells_height = last_in_each_cell_y.length

    if ((cells_width == 0) && (cells_height != 0)) || ((cells_width != 0) && (cells_height == 0))
      message = "Broken: divided #{png_file} into #{cells_width} by #{cells_height}"
      STDERR.puts message
    end

    if ((cells_width == 0) || (cells_height == 0))
      system "touch #{png_file}.empty"
    else

      puts "  Cropping from #{png_file}"

      crop_specification_filename = png_file + ".crop"
      open(crop_specification_filename,"w") do |f|

        c_number = 0
        0.upto( cells_width - 1 ) do |cell_x|
          0.upto( cells_height - 1 ) do |cell_y|
            right_x = Integer(last_in_each_cell_x[cell_x])
            left_x = Integer(first_in_each_cell_x[cell_x])
            top_y = Integer(first_in_each_cell_y[cell_y])
            bottom_y = Integer(last_in_each_cell_y[cell_y])
            output_filename = sprintf( png_output_template, c_number )
            unless FileTest.exist?( output_filename )
              f.puts( "#{output_filename} #{left_x} #{top_y} #{(right_x - left_x) + 1} #{(bottom_y - top_y) + 1}" )
            end
            c_number += 1
          end
        end
      end

      if not system("#{$helpers}/crop-images",png_file,crop_specification_filename)
        puts "#{$helpers}/crop-image #{png_file} #{crop_specification_filename} failed"
        exit(-1)
      end

    end
  end
end

$max_threads = java.lang.Runtime.getRuntime.availableProcessors
$mutex = Mutex.new
$condition = ConditionVariable.new
$threads = []
$running_threads = 0

unless FileTest.exist?( 'individual-characters/U-841-00000239.png' )

  png_files.each do |png_file|

    puts "Looking at png_file: #{png_file}"

    $threads << Thread.new( png_file ) { |file_to_split|

      find_grid_and_crop(file_to_split)

      $mutex.synchronize {
        $running_threads -= 1
        $condition.signal if $running_threads < $max_threads
      }
    }

  end

  $threads.each {|t| t.join}

end

puts "Done with threads..."


def tile_compare(a)
  a.gsub( /^.*U\-(\d+)\-(\d+)/, '\1\2' ).to_i(10)
end

files = Dir['individual-characters/U-*-*.png']

# If this is a repeat run then we may have some leftover -top and
# -bottom files here:

files.delete_if { |x| x =~ /top/ }
files.delete_if { |x| x =~ /bottom/ }

files = files.sort_by { |a| tile_compare(a) }

puts "Got files now..."

# ------------------------------------------------------------------------

def divide_and_ocr(character_filename)

  character_filename =~ /U-([0-9A-F]+)-([0-9A-F]+)/

  info = jruby_safe_backticks("#{$helpers}/png-size #{character_filename}",output_expected=true)

  width = nil
  height = nil

  info.chomp!
  if info =~ /(\d+)x(\d+)/

    width = Integer($1)
    height = Integer($2)

    top_part_fname = character_filename.sub( /.png/, '-top.png' )
    bottom_part_fname = character_filename.sub( /.png/, '-bottom.png' )

    guessed_text_size = 75

    if height < guessed_text_size
      message = "File too short: #{character_filename}"
      STDERR.puts message
      $mutex.synchronize {
        open('splitting.log','a') { |f| f.puts message }
      }
    else

      text_starts_at = height - guessed_text_size

      t = Tempfile.new('unicode-tile')
      t.puts( "#{top_part_fname} 0 0 #{width} #{text_starts_at}" )
      t.puts( "#{bottom_part_fname} 0 #{text_starts_at} #{width} #{guessed_text_size}" )
      t.close()

      if not system("#{$helpers}/crop-images",character_filename,t.path)
        puts "#{$helpers}/crop-image #{character_filename} #{t.path} failed"
        exit(-1)
      end

      top_width = nil
      top_height = nil

      output = jruby_safe_backticks("#{$helpers}/empty-image #{top_part_fname}",output_expected=true)
      output.chomp!
      if output == "Empty"
        puts "Empty, skipping: #{character_filename}"
        system("rm",top_part_fname)
        system("rm",bottom_part_fname)
      else
        output.chomp!
        if output =~ /^(\d+) (\d+)/
          top_width = Integer($1)
          top_height = Integer($2)
        end

        # Check that the bottom part isn't hashed out, and delete
        # it if so:

        proportion = nil
        output = jruby_safe_backticks("#{$helpers}/bottom-line-proportion #{bottom_part_fname}",output_expected=true)
        begin
          proportion = Float(output)
        rescue ArgumentError => e
          STDERR.puts "Parsing a proportion out of the output '#{output}' from file #{bottom_part_fname} failed"
        end
        if proportion and proportion > 0.01
          puts "Probably cross-hatched, skipping: #{character_filename}"
          system("rm",top_part_fname)
          system("rm",bottom_part_fname)
        else

          tileinfo = TileInfo.new(top_width,top_height)
          tileinfo.filename = top_part_fname

          # Now use OCR to try to parse the codepoint out of the bottom
          # part:

          result = jruby_safe_backticks("pngtopnm #{bottom_part_fname} | ocrad -",output_expected=true)

          result.chomp!
          result.gsub!( /\s/, '' )
          result.gsub!( /[oO]/, '0' )
          result.gsub!( /a/, '8' )
          result.gsub!( /g/, '9' )

          result.upcase!

          tileinfo.ocr_codepoint = "0x" + result
          return tileinfo
        end
      end
    end
  else
    STDERR.puts "'#{$helpers}/png-size #{character_filename}' failed"
  end
  nil
end

$threads = []
$running_threads = 0

top_sizes = []
codepoints = []

class Codepoint
  attr_accessor :filename, :codepoint
  def initialize(f,c)
    @filename = f
    @codepoint = c
  end
end

$max_threads *= 2

done = 0

files.each do |fname|

  puts "Considering for OCR #{fname}"

  # break if done > 100

  $mutex.synchronize {
    $condition.wait($mutex) while $running_threads >= $max_threads
    $running_threads += 1
    $condition.signal if $running_threads < $max_threads
  }

  thread = Thread.new(fname) { |file_to_process|

    Thread.current[:tileinfo] = divide_and_ocr(file_to_process)

    $mutex.synchronize {
      if Thread.current[:tileinfo]
        top_sizes <<  Thread.current[:tileinfo]
      end

      $running_threads -= 1
      $condition.signal if $running_threads < $max_threads
    }
  }

  # Go through the list of threads, find any that have finished, join
  # them, and remove all the finished threads from the array afterwards:
  $mutex.synchronize {
    indices_to_remove = []
    $threads.each_index do |i|
      if $threads[i]
        status = $threads[i].status
        if status == false or status == nil
          $threads[i].join
          indices_to_remove << i
        end
      end
    end
    indices_to_remove.reverse.each do |i|
      $threads.delete_at i
    end
    $threads << thread
  }

  done += 1

end

$threads.each {|t| t.join}

open( "top-sizes.yaml", "w" ) do |o_top_sizes|
  o_top_sizes.puts "---"
  top_sizes.each do |t|
    o_top_sizes.puts(t.to_yaml_hash_element)
  end
end

open( "codepoints.yaml", "w" ) do |o_codepoints|
  top_sizes.each do |t|
    o_codepoints.puts "-"
    o_codepoints.puts '  - "' + t.filename + '"'
    o_codepoints.puts "  - " + t.ocr_codepoint
  end
end