[DRE-commits] [ruby-parallel] 06/13: New upstream version 1.9.0
Cédric Boutillier
boutil at moszumanska.debian.org
Sun Dec 25 23:23:25 UTC 2016
This is an automated email from the git hooks/post-receive script.
boutil pushed a commit to branch master
in repository ruby-parallel.
commit 4f062bc192b1c8d00d604838e6ac59f5078d3c0c
Author: Cédric Boutillier <boutil at debian.org>
Date: Sat Nov 19 01:47:38 2016 +0100
New upstream version 1.9.0
---
.gitignore | 1 +
.travis.yml | 14 +
Gemfile | 13 +
Gemfile.lock | 74 +++
Rakefile | 7 +
Readme.md | 163 ++++++
gem-public_cert.pem | 21 +
parallel.gemspec | 34 +-
spec/cases/after_interrupt.rb | 7 +
spec/cases/closes_processes_at_runtime.rb | 5 +
spec/cases/count_open_pipes.rb | 6 +
spec/cases/double_interrupt.rb | 13 +
spec/cases/each.rb | 8 +
spec/cases/each_in_place.rb | 6 +
spec/cases/each_with_ar_sqlite.rb | 36 ++
spec/cases/each_with_index.rb | 5 +
spec/cases/eof_in_process.rb | 9 +
spec/cases/exit_in_process.rb | 9 +
spec/cases/fatal_queue.rb | 7 +
spec/cases/helper.rb | 20 +
spec/cases/map_isolation.rb | 8 +
spec/cases/map_with_ar.rb | 45 ++
spec/cases/map_with_index.rb | 6 +
spec/cases/map_with_index_empty.rb | 6 +
spec/cases/map_with_killed_worker_before_read.rb | 9 +
spec/cases/map_with_killed_worker_before_write.rb | 18 +
spec/cases/map_with_nested_arrays_and_nil.rb | 7 +
spec/cases/map_worker_number_isolation.rb | 8 +
spec/cases/no_dump_with_each.rb | 21 +
spec/cases/no_gc_with_each.rb | 5 +
spec/cases/parallel_fast_exit.rb | 7 +
spec/cases/parallel_high_fork_rate.rb | 6 +
spec/cases/parallel_influence_outside_data.rb | 8 +
spec/cases/parallel_kill.rb | 15 +
spec/cases/parallel_map.rb | 6 +
spec/cases/parallel_map_complex_objects.rb | 8 +
spec/cases/parallel_map_range.rb | 6 +
spec/cases/parallel_map_sleeping.rb | 5 +
spec/cases/parallel_map_uneven.rb | 5 +
spec/cases/parallel_raise.rb | 10 +
spec/cases/parallel_raise_undumpable.rb | 17 +
spec/cases/parallel_sleeping_2.rb | 5 +
spec/cases/parallel_start_and_kill.rb | 19 +
spec/cases/parallel_with_detected_cpus.rb | 6 +
spec/cases/parallel_with_nil_uses_detected_cpus.rb | 6 +
spec/cases/parallel_with_set_processes.rb | 6 +
spec/cases/profile_memroy.rb | 23 +
spec/cases/progress.rb | 6 +
spec/cases/progress_with_finish.rb | 11 +
spec/cases/progress_with_options.rb | 27 +
spec/cases/synchronizes_start_and_finish.rb | 16 +
spec/cases/with_break.rb | 13 +
spec/cases/with_break_before_finish.rb | 19 +
spec/cases/with_exception.rb | 16 +
spec/cases/with_exception_before_finish.rb | 23 +
spec/cases/with_exception_in_finish.rb | 19 +
spec/cases/with_exception_in_start.rb | 19 +
.../cases/with_exception_in_start_before_finish.rb | 26 +
spec/cases/with_lambda.rb | 12 +
spec/cases/with_queue.rb | 18 +
spec/cases/with_worker_number.rb | 10 +
spec/parallel_spec.rb | 554 +++++++++++++++++++++
spec/spec_helper.rb | 7 +
63 files changed, 1519 insertions(+), 26 deletions(-)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b33679f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+/rspec.failures
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..7cf409d
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,14 @@
+language: ruby
+cache: bundler
+sudo: false
+branches:
+ only: master
+matrix:
+ fast_finish: true
+rvm:
+ - 1.9.3
+ - 2.0.0
+ - 2.1.6
+ - 2.2.4
+ # - 2.3.0 # 1 test fails with 1 extra process
+
diff --git a/Gemfile b/Gemfile
new file mode 100644
index 0000000..805414e
--- /dev/null
+++ b/Gemfile
@@ -0,0 +1,13 @@
+source "https://rubygems.org"
+gemspec
+
+gem 'bump'
+gem 'rake'
+gem 'rspec'
+gem 'activerecord'
+gem 'ruby-progressbar'
+gem 'rspec-rerun'
+gem 'rspec-legacy_formatters'
+
+gem 'mysql2', :group => :mysql
+gem 'sqlite3'
diff --git a/Gemfile.lock b/Gemfile.lock
new file mode 100644
index 0000000..3d97bd5
--- /dev/null
+++ b/Gemfile.lock
@@ -0,0 +1,74 @@
+PATH
+ remote: .
+ specs:
+ parallel (1.9.0)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activemodel (4.2.1)
+ activesupport (= 4.2.1)
+ builder (~> 3.1)
+ activerecord (4.2.1)
+ activemodel (= 4.2.1)
+ activesupport (= 4.2.1)
+ arel (~> 6.0)
+ activesupport (4.2.1)
+ i18n (~> 0.7)
+ json (~> 1.7, >= 1.7.7)
+ minitest (~> 5.1)
+ thread_safe (~> 0.3, >= 0.3.4)
+ tzinfo (~> 1.1)
+ arel (6.0.0)
+ builder (3.2.2)
+ bump (0.5.2)
+ diff-lcs (1.2.5)
+ i18n (0.7.0)
+ json (1.8.2)
+ json (1.8.2-java)
+ minitest (5.6.1)
+ mysql2 (0.3.18)
+ rake (10.4.2)
+ rspec (3.2.0)
+ rspec-core (~> 3.2.0)
+ rspec-expectations (~> 3.2.0)
+ rspec-mocks (~> 3.2.0)
+ rspec-core (3.2.3)
+ rspec-support (~> 3.2.0)
+ rspec-expectations (3.2.1)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.2.0)
+ rspec-legacy_formatters (1.0.0)
+ rspec-core (>= 3.0.0.beta2)
+ rspec-support (>= 3.0.0.beta2)
+ rspec-mocks (3.2.1)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.2.0)
+ rspec-rerun (0.3.0)
+ rspec
+ rspec-support (3.2.2)
+ ruby-progressbar (1.7.5)
+ sqlite3 (1.3.11)
+ thread_safe (0.3.5)
+ thread_safe (0.3.5-java)
+ tzinfo (1.2.2)
+ thread_safe (~> 0.1)
+
+PLATFORMS
+ java
+ ruby
+
+DEPENDENCIES
+ activerecord
+ bump
+ mysql2
+ parallel!
+ rake
+ rspec
+ rspec-legacy_formatters
+ rspec-rerun
+ ruby-progressbar
+ sqlite3
+
+BUNDLED WITH
+ 1.12.3
diff --git a/Rakefile b/Rakefile
new file mode 100644
index 0000000..f0388af
--- /dev/null
+++ b/Rakefile
@@ -0,0 +1,7 @@
+require 'bundler/setup'
+require 'bundler/gem_tasks'
+require 'bump/tasks'
+require "rspec/core/rake_task"
+require "rspec-rerun"
+
+task :default => "rspec-rerun:spec"
diff --git a/Readme.md b/Readme.md
new file mode 100644
index 0000000..8402a9a
--- /dev/null
+++ b/Readme.md
@@ -0,0 +1,163 @@
+Run any code in parallel Processes(> use all CPUs) or Threads(> speedup blocking operations).<br/>
+Best suited for map-reduce or e.g. parallel downloads/uploads.
+
+Install
+=======
+
+```Bash
+gem install parallel
+```
+
+Usage
+=====
+
+```Ruby
+# 2 CPUs -> work in 2 processes (a,b + c)
+results = Parallel.map(['a','b','c']) do |one_letter|
+ expensive_calculation(one_letter)
+end
+
+# 3 Processes -> finished after 1 run
+results = Parallel.map(['a','b','c'], in_processes: 3) { |one_letter| ... }
+
+# 3 Threads -> finished after 1 run
+results = Parallel.map(['a','b','c'], in_threads: 3) { |one_letter| ... }
+```
+
+Same can be done with `each`
+```Ruby
+Parallel.each(['a','b','c']) { |one_letter| ... }
+```
+or `each_with_index` or `map_with_index`
+
+Produce one item at a time with `lambda` (anything that responds to `.call`) or `Queue`.
+
+```Ruby
+items = [1,2,3]
+Parallel.each( -> { items.pop || Parallel::Stop }) { |number| ... }
+```
+
+
+Processes/Threads are workers, they grab the next piece of work when they finish.
+
+### Processes
+ - Speedup through multiple CPUs
+ - Speedup for blocking operations
+ - Protects global data
+ - Extra memory used
+ - Child processes are killed when your main process is killed through Ctrl+c or kill -2
+
+### Threads
+ - Speedup for blocking operations
+ - Global data can be modified
+ - No extra memory used
+
+### ActiveRecord
+
+Try any of those to get working parallel AR
+
+```Ruby
+# reproducibly fixes things (spec/cases/map_with_ar.rb)
+Parallel.each(User.all, in_processes: 8) do |user|
+ user.update_attribute(:some_attribute, some_value)
+end
+User.connection.reconnect!
+
+# maybe helps: explicitly use connection pool
+Parallel.each(User.all, in_threads: 8) do |user|
+ ActiveRecord::Base.connection_pool.with_connection do
+ user.update_attribute(:some_attribute, some_value)
+ end
+end
+
+# maybe helps: reconnect once inside every fork
+Parallel.each(User.all, in_processes: 8) do |user|
+ @reconnected ||= User.connection.reconnect! || true
+ user.update_attribute(:some_attribute, some_value)
+end
+```
+
+### Break
+
+```Ruby
+Parallel.map(User.all) do |user|
+ raise Parallel::Break # -> stops after all current items are finished
+end
+```
+
+### Kill
+
+Only use if whatever is executing in the sub-command is safe to kill at any point
+
+```Ruby
+Parallel.map([1,2,3]) do |x|
+ raise Parallel::Kill if x == 1# -> stop all sub-processes, killing them instantly
+ sleep 100
+end
+```
+
+### Progress / ETA
+
+```Ruby
+# gem install ruby-progressbar
+
+Parallel.map(1..50, progress: "Doing stuff") { sleep 1 }
+
+# Doing stuff | ETA: 00:00:02 | ==================== | Time: 00:00:10
+```
+
+Use `:finish` or `:start` hook to get progress information.
+ - `:start` has item and index
+ - `:finish` has item, index, result
+
+They are called on the main process and protected with a mutex.
+
+```Ruby
+Parallel.map(1..100, finish: -> (item, i, result) { ... do something ... }) { sleep 1 }
+```
+
+Tips
+====
+ - [Benchmark/Test] Disable threading/forking with `in_threads: 0` or `in_processes: 0`, great to test performance or to debug parallel issues
+ - [Isolation] Do not reuse previous worker processes: `isolation: true`
+
+TODO
+====
+ - Replace Signal trapping with simple `rescue Interrupt` handler
+
+Authors
+=======
+
+### [Contributors](https://github.com/grosser/parallel/graphs/contributors)
+ - [Przemyslaw Wroblewski](https://github.com/lowang)
+ - [TJ Holowaychuk](http://vision-media.ca/)
+ - [Masatomo Nakano](https://twitter.com/masatomo2)
+ - [Fred Wu](http://fredwu.me)
+ - [mikezter](https://github.com/mikezter)
+ - [Jeremy Durham](http://www.jeremydurham.com)
+ - [Nick Gauthier](http://www.ngauthier.com)
+ - [Andrew Bowerman](http://andrewbowerman.com)
+ - [Byron Bowerman](http://blog.bm5k.com/)
+ - [Mikko Kokkonen](https://github.com/mikian)
+ - [brian p o'rourke](https://github.com/bpo)
+ - [Norio Sato]
+ - [Neal Stewart](https://github.com/n-time)
+ - [Jurriaan Pruis](https://github.com/jurriaan)
+ - [Rob Worley](https://github.com/robworley)
+ - [Tasveer Singh](https://github.com/tazsingh)
+ - [Joachim](https://github.com/jmozmoz)
+ - [yaoguai](https://github.com/yaoguai)
+ - [Bartosz Dziewoński](https://github.com/MatmaRex)
+ - [yaoguai](https://github.com/yaoguai)
+ - [Guillaume Hain](https://github.com/zedtux)
+ - [Adam Wróbel](https://github.com/amw)
+ - [Matthew Brennan](https://github.com/mattyb)
+ - [Brendan Dougherty](https://github.com/brendar)
+ - [Daniel Finnie](https://github.com/danfinnie)
+ - [Philip M. White](https://github.com/philipmw)
+ - [Arlan Jaska](https://github.com/ajaska)
+
+[Michael Grosser](http://grosser.it)<br/>
+michael at grosser.it<br/>
+License: MIT<br/>
+[](https://travis-ci.org/grosser/parallel)
diff --git a/gem-public_cert.pem b/gem-public_cert.pem
new file mode 100644
index 0000000..804fc62
--- /dev/null
+++ b/gem-public_cert.pem
@@ -0,0 +1,21 @@
+-----BEGIN CERTIFICATE-----
+MIIDcDCCAligAwIBAgIBATANBgkqhkiG9w0BAQUFADA/MRAwDgYDVQQDDAdtaWNo
+YWVsMRcwFQYKCZImiZPyLGQBGRYHZ3Jvc3NlcjESMBAGCgmSJomT8ixkARkWAml0
+MB4XDTE0MDIwNDIwMjk0MVoXDTE1MDIwNDIwMjk0MVowPzEQMA4GA1UEAwwHbWlj
+aGFlbDEXMBUGCgmSJomT8ixkARkWB2dyb3NzZXIxEjAQBgoJkiaJk/IsZAEZFgJp
+dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMorXo/hgbUq97+kII9H
+MsQcLdC/7wQ1ZP2OshVHPkeP0qH8MBHGg6eYisOX2ubNagF9YTCZWnhrdKrwpLOO
+cPLaZbjUjljJ3cQR3B8Yn1veV5IhG86QseTBjymzJWsLpqJ1UZGpfB9tXcsFtuxO
+6vHvcIHdzvc/OUkICttLbH+1qb6rsHUceqh+JrH4GrsJ5H4hAfIdyS2XMK7YRKbh
+h+IBu6dFWJJByzFsYmV1PDXln3UBmgAt65cmCu4qPfThioCGDzbSJrGDGLmw/pFX
+FPpVCm1zgYSb1v6Qnf3cgXa2f2wYGm17+zAVyIDpwryFru9yF/jJxE38z/DRsd9R
+/88CAwEAAaN3MHUwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0OBBYEFLIj
+Z1x7SnjGGHK+MiVZkFjjS/iMMB0GA1UdEQQWMBSBEm1pY2hhZWxAZ3Jvc3Nlci5p
+dDAdBgNVHRIEFjAUgRJtaWNoYWVsQGdyb3NzZXIuaXQwDQYJKoZIhvcNAQEFBQAD
+ggEBAExBcUWfGuamYn+IddOA0Ws8jUKwB14RXoZRDrTiTAlMm3Bkg2OKyxS3uJXa
+6Z+LwFiZwVYk62yHXqNzEJycQk4SEmY+xDWLj0p7X6qEeU4QZKwR1TwJ5z3PTrZ6
+irJgM3q7NIBRvmTzRaAghWcQn+Eyr5YLOfMksjVBMUMnzh5/ZDgq53LphgJbGwvz
+ScJAgfNclLHnjk9q1mT1s0e1FPWbiAL3siBIR5HpH8qtSEiivTf2ntciebOqS93f
+F5etKHZg0j3eHO31/i2HnswY04lqGImUu6aM5EnijFTB7PPW2KwKKM4+kKDYFdlw
+/0WV1Ng2/Y6qsHwmqGg2VlYj2h4=
+-----END CERTIFICATE-----
diff --git a/parallel.gemspec b/parallel.gemspec
index e6e67e4..d2ba2d9 100644
--- a/parallel.gemspec
+++ b/parallel.gemspec
@@ -1,30 +1,12 @@
-#########################################################
-# This file has been automatically generated by gem2tgz #
-#########################################################
-# -*- encoding: utf-8 -*-
+name = "parallel"
+require "./lib/#{name}/version"
-Gem::Specification.new do |s|
- s.name = "parallel"
- s.version = "1.9.0"
-
- s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+Gem::Specification.new name, Parallel::VERSION do |s|
+ s.summary = "Run any kind of code in parallel processes"
s.authors = ["Michael Grosser"]
- s.date = "2016-06-03"
s.email = "michael at grosser.it"
- s.files = ["MIT-LICENSE.txt", "lib/parallel.rb", "lib/parallel/processor_count.rb", "lib/parallel/version.rb"]
- s.homepage = "https://github.com/grosser/parallel"
- s.licenses = ["MIT"]
- s.require_paths = ["lib"]
- s.required_ruby_version = Gem::Requirement.new(">= 1.9.3")
- s.rubygems_version = "1.8.23"
- s.summary = "Run any kind of code in parallel processes"
-
- if s.respond_to? :specification_version then
- s.specification_version = 4
-
- if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
- else
- end
- else
- end
+ s.homepage = "https://github.com/grosser/#{name}"
+ s.files = `git ls-files lib MIT-LICENSE.txt`.split("\n")
+ s.license = "MIT"
+ s.required_ruby_version = '>= 1.9.3'
end
diff --git a/spec/cases/after_interrupt.rb b/spec/cases/after_interrupt.rb
new file mode 100644
index 0000000..fef31f7
--- /dev/null
+++ b/spec/cases/after_interrupt.rb
@@ -0,0 +1,7 @@
+require './spec/cases/helper'
+
+Parallel.map([1, 2], :in_processes => 2) { }
+
+puts Signal.trap(:SIGINT, "IGNORE")
+
+
diff --git a/spec/cases/closes_processes_at_runtime.rb b/spec/cases/closes_processes_at_runtime.rb
new file mode 100644
index 0000000..f45e86d
--- /dev/null
+++ b/spec/cases/closes_processes_at_runtime.rb
@@ -0,0 +1,5 @@
+require './spec/cases/helper'
+
+process_diff do
+ Parallel.each((0..10).to_a, :in_processes => 5) { |a| a*2 }
+end
diff --git a/spec/cases/count_open_pipes.rb b/spec/cases/count_open_pipes.rb
new file mode 100644
index 0000000..f7a5446
--- /dev/null
+++ b/spec/cases/count_open_pipes.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+results = Parallel.map(Array.new(20), :in_processes => 20) do
+ `lsof | grep pipe | wc -l`.to_i
+end
+puts results.max
diff --git a/spec/cases/double_interrupt.rb b/spec/cases/double_interrupt.rb
new file mode 100644
index 0000000..5541c44
--- /dev/null
+++ b/spec/cases/double_interrupt.rb
@@ -0,0 +1,13 @@
+require './spec/cases/helper'
+
+Signal.trap :SIGINT do
+ sleep 0.5
+ puts "YES"
+ exit 0
+end
+
+Parallel.map(Array.new(20), :in_processes => 2) do
+ sleep 10
+ puts "I should be killed earlier"
+end
+
diff --git a/spec/cases/each.rb b/spec/cases/each.rb
new file mode 100644
index 0000000..2db871d
--- /dev/null
+++ b/spec/cases/each.rb
@@ -0,0 +1,8 @@
+require './spec/cases/helper'
+STDOUT.sync = true # otherwise results can go weird...
+
+x = ['a','b','c','d']
+result = Parallel.each(x) do |x|
+ sleep 0.1 if x == 'a'
+end
+print result * ' '
diff --git a/spec/cases/each_in_place.rb b/spec/cases/each_in_place.rb
new file mode 100644
index 0000000..68d3064
--- /dev/null
+++ b/spec/cases/each_in_place.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+STDOUT.sync = true # otherwise results can go weird...
+
+x = ['a']
+Parallel.each(x, :in_threads => 1) { |x| x << 'b' }
+print x.first
diff --git a/spec/cases/each_with_ar_sqlite.rb b/spec/cases/each_with_ar_sqlite.rb
new file mode 100644
index 0000000..be85a27
--- /dev/null
+++ b/spec/cases/each_with_ar_sqlite.rb
@@ -0,0 +1,36 @@
+require './spec/cases/helper'
+require "active_record"
+require "sqlite3"
+STDOUT.sync = true
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+ActiveRecord::Schema.verbose = false
+ActiveRecord::Base.establish_connection(
+ :adapter => "sqlite3",
+ :database => Tempfile.new("db").path
+)
+
+class User < ActiveRecord::Base
+end
+
+# create tables
+unless User.table_exists?
+ ActiveRecord::Schema.define(:version => 1) do
+ create_table :users do |t|
+ t.string :name
+ end
+ end
+end
+
+User.delete_all
+
+3.times { User.create!(:name => "X") }
+
+puts "Parent: #{User.first.name}"
+
+Parallel.each([1], in_worker_type => 1) do
+ puts "Parallel (#{in_worker_type}): #{User.all.map(&:name).join}"
+end
+
+puts "Parent: #{User.first.name}"
+
diff --git a/spec/cases/each_with_index.rb b/spec/cases/each_with_index.rb
new file mode 100644
index 0000000..2f61fe6
--- /dev/null
+++ b/spec/cases/each_with_index.rb
@@ -0,0 +1,5 @@
+require './spec/cases/helper'
+
+Parallel.each_with_index(['a','b'], :in_threads => 2) do |x, i|
+ print "#{x}#{i}"
+end
diff --git a/spec/cases/eof_in_process.rb b/spec/cases/eof_in_process.rb
new file mode 100644
index 0000000..7380fbd
--- /dev/null
+++ b/spec/cases/eof_in_process.rb
@@ -0,0 +1,9 @@
+require './spec/cases/helper'
+
+begin
+ Parallel.map([1]){ raise EOFError }
+rescue EOFError
+ puts 'Yep, EOF'
+else
+ puts 'WHOOOPS'
+end
diff --git a/spec/cases/exit_in_process.rb b/spec/cases/exit_in_process.rb
new file mode 100644
index 0000000..549d118
--- /dev/null
+++ b/spec/cases/exit_in_process.rb
@@ -0,0 +1,9 @@
+require './spec/cases/helper'
+
+begin
+ Parallel.map([1]){ exit }
+rescue Parallel::DeadWorker
+ puts "Yep, DEAD"
+else
+ puts "WHOOOPS"
+end
diff --git a/spec/cases/fatal_queue.rb b/spec/cases/fatal_queue.rb
new file mode 100644
index 0000000..7d2a17d
--- /dev/null
+++ b/spec/cases/fatal_queue.rb
@@ -0,0 +1,7 @@
+require './spec/cases/helper'
+
+queue = Queue.new
+queue.push 1
+queue.push 2
+queue.push 3
+Parallel.map(queue, :in_threads => 2) { |(i, id)| "ITEM-#{i}" }
diff --git a/spec/cases/helper.rb b/spec/cases/helper.rb
new file mode 100644
index 0000000..ce9fc73
--- /dev/null
+++ b/spec/cases/helper.rb
@@ -0,0 +1,20 @@
+require 'bundler/setup'
+require 'parallel'
+
+def process_diff
+ cmd = "ps uaxw|grep ruby|wc -l"
+
+ processes_before = `#{cmd}`.to_i
+
+ yield
+
+ sleep 1
+
+ processes_after = `#{cmd}`.to_i
+
+ if processes_before == processes_after
+ print 'OK'
+ else
+ print "FAIL: before:#{processes_before} -- after:#{processes_after}"
+ end
+end
diff --git a/spec/cases/map_isolation.rb b/spec/cases/map_isolation.rb
new file mode 100644
index 0000000..0bed5b7
--- /dev/null
+++ b/spec/cases/map_isolation.rb
@@ -0,0 +1,8 @@
+require './spec/cases/helper'
+
+process_diff do
+ result = Parallel.map([1,2,3,4], in_processes: 2, isolation: true) do |i|
+ $i ||= i
+ end
+ puts result
+end
diff --git a/spec/cases/map_with_ar.rb b/spec/cases/map_with_ar.rb
new file mode 100644
index 0000000..b08d052
--- /dev/null
+++ b/spec/cases/map_with_ar.rb
@@ -0,0 +1,45 @@
+require './spec/cases/helper'
+require "active_record"
+
+Tempfile.open("xxx") do |f|
+ database = "parallel_with_ar_test"
+ `mysql #{database} -e '' || mysql -e 'create database #{database}'`
+
+ ActiveRecord::Schema.verbose = false
+ ActiveRecord::Base.establish_connection(
+ :adapter => "mysql2",
+ :database => database
+ )
+
+ class User < ActiveRecord::Base
+ end
+
+ # create tables
+ unless User.table_exists?
+ ActiveRecord::Schema.define(:version => 1) do
+ create_table :users do |t|
+ t.string :name
+ end
+ end
+ end
+
+ User.delete_all
+
+ User.create!(:name => "X")
+
+ Parallel.map(1..8) do |i|
+ User.create!(:name => i)
+ end
+
+ puts "User.count: #{User.count}"
+
+ puts User.connection.reconnect!.inspect
+
+ Parallel.map(1..8, :in_threads => 4) do |i|
+ User.create!(:name => i)
+ end
+
+ User.create!(:name => "X")
+
+ puts User.all.map(&:name).sort.join("-")
+end
diff --git a/spec/cases/map_with_index.rb b/spec/cases/map_with_index.rb
new file mode 100644
index 0000000..a5d66a9
--- /dev/null
+++ b/spec/cases/map_with_index.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+result = Parallel.map_with_index(['a','b']) do |x, i|
+ "#{x}#{i}"
+end
+print result * ''
diff --git a/spec/cases/map_with_index_empty.rb b/spec/cases/map_with_index_empty.rb
new file mode 100644
index 0000000..733a588
--- /dev/null
+++ b/spec/cases/map_with_index_empty.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+result = Parallel.map_with_index([]) do |x, i|
+ "#{x}#{i}"
+end
+print result * ''
diff --git a/spec/cases/map_with_killed_worker_before_read.rb b/spec/cases/map_with_killed_worker_before_read.rb
new file mode 100644
index 0000000..306e8ad
--- /dev/null
+++ b/spec/cases/map_with_killed_worker_before_read.rb
@@ -0,0 +1,9 @@
+require './spec/cases/helper'
+
+begin
+ Parallel.map([1,2,3]) do |x, i|
+ Process.kill("SIGKILL", Process.pid)
+ end
+rescue Parallel::DeadWorker
+ puts "DEAD"
+end
diff --git a/spec/cases/map_with_killed_worker_before_write.rb b/spec/cases/map_with_killed_worker_before_write.rb
new file mode 100644
index 0000000..29cc358
--- /dev/null
+++ b/spec/cases/map_with_killed_worker_before_write.rb
@@ -0,0 +1,18 @@
+require './spec/cases/helper'
+
+Parallel::Worker.class_eval do
+ alias_method :work_without_kill, :work
+ def work(*args)
+ Process.kill("SIGKILL", pid)
+ sleep 0.5
+ work_without_kill(*args)
+ end
+end
+
+begin
+ Parallel.map([1,2,3]) do |x, i|
+ Process.kill("SIGKILL", Process.pid)
+ end
+rescue Parallel::DeadWorker
+ puts "DEAD"
+end
diff --git a/spec/cases/map_with_nested_arrays_and_nil.rb b/spec/cases/map_with_nested_arrays_and_nil.rb
new file mode 100644
index 0000000..1c934ba
--- /dev/null
+++ b/spec/cases/map_with_nested_arrays_and_nil.rb
@@ -0,0 +1,7 @@
+require './spec/cases/helper'
+
+result = Parallel.map([1,2,[3]]) do |x|
+ [x, x] if x != 1
+end
+
+print result.inspect
diff --git a/spec/cases/map_worker_number_isolation.rb b/spec/cases/map_worker_number_isolation.rb
new file mode 100644
index 0000000..0f54fda
--- /dev/null
+++ b/spec/cases/map_worker_number_isolation.rb
@@ -0,0 +1,8 @@
+require './spec/cases/helper'
+
+process_diff do
+ result = Parallel.map([1,2,3,4], in_processes: 2, isolation: true) do |i|
+ Parallel.worker_number
+ end
+ puts result.uniq.sort.join(',')
+end
diff --git a/spec/cases/no_dump_with_each.rb b/spec/cases/no_dump_with_each.rb
new file mode 100644
index 0000000..8437aa8
--- /dev/null
+++ b/spec/cases/no_dump_with_each.rb
@@ -0,0 +1,21 @@
+require './spec/cases/helper'
+
+class NotDumpable
+ def marshal_dump
+ raise "NOOOO"
+ end
+
+ def to_s
+ 'not dumpable'
+ end
+end
+
+Parallel.each([1]) do
+ print 'no dump for result'
+ NotDumpable.new
+end
+
+Parallel.each([NotDumpable.new]) do
+ print 'no dump for each'
+ 1
+end
diff --git a/spec/cases/no_gc_with_each.rb b/spec/cases/no_gc_with_each.rb
new file mode 100644
index 0000000..524be85
--- /dev/null
+++ b/spec/cases/no_gc_with_each.rb
@@ -0,0 +1,5 @@
+require './spec/cases/helper'
+
+Parallel.each(1..1000, :in_threads => 2) do |i|
+ "xxxx" * 1_000_000
+end
diff --git a/spec/cases/parallel_fast_exit.rb b/spec/cases/parallel_fast_exit.rb
new file mode 100644
index 0000000..4deb827
--- /dev/null
+++ b/spec/cases/parallel_fast_exit.rb
@@ -0,0 +1,7 @@
+require './spec/cases/helper'
+
+Parallel.map([1,2,3], :in_processes => 2) do
+ puts "I finished..."
+end
+
+sleep 10
diff --git a/spec/cases/parallel_high_fork_rate.rb b/spec/cases/parallel_high_fork_rate.rb
new file mode 100644
index 0000000..93b4487
--- /dev/null
+++ b/spec/cases/parallel_high_fork_rate.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+Parallel.each((0..200).to_a, :in_processes=>200) do |x|
+ sleep 1
+end
+print 'OK'
diff --git a/spec/cases/parallel_influence_outside_data.rb b/spec/cases/parallel_influence_outside_data.rb
new file mode 100644
index 0000000..5a63369
--- /dev/null
+++ b/spec/cases/parallel_influence_outside_data.rb
@@ -0,0 +1,8 @@
+require './spec/cases/helper'
+
+x = 'yes'
+
+Parallel.in_processes(2) do
+ x = 'no'
+end
+print x
diff --git a/spec/cases/parallel_kill.rb b/spec/cases/parallel_kill.rb
new file mode 100644
index 0000000..03f4f0b
--- /dev/null
+++ b/spec/cases/parallel_kill.rb
@@ -0,0 +1,15 @@
+require './spec/cases/helper'
+
+results = Parallel.map([1,2,3]) do |x|
+ if x == 1 # -> stop all sub-processes, killing them instantly
+ sleep 0.1
+ puts "DEAD"
+ raise Parallel::Kill
+ elsif x == 3
+ sleep 10
+ else
+ x
+ end
+end
+
+puts "Works #{results.inspect}"
diff --git a/spec/cases/parallel_map.rb b/spec/cases/parallel_map.rb
new file mode 100644
index 0000000..4e149ac
--- /dev/null
+++ b/spec/cases/parallel_map.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+result = Parallel.map(['a','b','c','d']) do |x|
+ "-#{x}-"
+end
+print result * ' '
diff --git a/spec/cases/parallel_map_complex_objects.rb b/spec/cases/parallel_map_complex_objects.rb
new file mode 100644
index 0000000..f8e854f
--- /dev/null
+++ b/spec/cases/parallel_map_complex_objects.rb
@@ -0,0 +1,8 @@
+require './spec/cases/helper'
+
+object = ["\nasd#{File.read('Gemfile')}--#{File.read('Rakefile')}"*100, 12345, {:b=>:a}]
+
+result = Parallel.map([1,2]) do |x|
+ object
+end
+print 'YES' if result.inspect == [object, object].inspect
diff --git a/spec/cases/parallel_map_range.rb b/spec/cases/parallel_map_range.rb
new file mode 100644
index 0000000..177c7fe
--- /dev/null
+++ b/spec/cases/parallel_map_range.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+result = Parallel.map(1..5) do |x|
+ x
+end
+print result.inspect
diff --git a/spec/cases/parallel_map_sleeping.rb b/spec/cases/parallel_map_sleeping.rb
new file mode 100644
index 0000000..da437c3
--- /dev/null
+++ b/spec/cases/parallel_map_sleeping.rb
@@ -0,0 +1,5 @@
+require './spec/cases/helper'
+
+Parallel.map(['a','b','c','d']) do |x|
+ sleep 1
+end
diff --git a/spec/cases/parallel_map_uneven.rb b/spec/cases/parallel_map_uneven.rb
new file mode 100644
index 0000000..8df7533
--- /dev/null
+++ b/spec/cases/parallel_map_uneven.rb
@@ -0,0 +1,5 @@
+require './spec/cases/helper'
+
+Parallel.map([1,2,1,2]) do |x|
+ sleep 2 if x == 1
+end
diff --git a/spec/cases/parallel_raise.rb b/spec/cases/parallel_raise.rb
new file mode 100644
index 0000000..a416af6
--- /dev/null
+++ b/spec/cases/parallel_raise.rb
@@ -0,0 +1,10 @@
+require './spec/cases/helper'
+
+begin
+ Parallel.in_processes(2) do
+ raise "TEST"
+ end
+ puts "FAIL"
+rescue RuntimeError
+ puts $!.message
+end
diff --git a/spec/cases/parallel_raise_undumpable.rb b/spec/cases/parallel_raise_undumpable.rb
new file mode 100644
index 0000000..54ec622
--- /dev/null
+++ b/spec/cases/parallel_raise_undumpable.rb
@@ -0,0 +1,17 @@
+require './spec/cases/helper'
+require 'stringio'
+
+class MyException < StandardError
+ def initialize(object)
+ @object = object
+ end
+end
+
+begin
+ Parallel.in_processes(2) do
+ raise MyException.new(StringIO.new)
+ end
+ puts "FAIL"
+rescue RuntimeError
+ puts $!.message
+end
diff --git a/spec/cases/parallel_sleeping_2.rb b/spec/cases/parallel_sleeping_2.rb
new file mode 100644
index 0000000..e5281e4
--- /dev/null
+++ b/spec/cases/parallel_sleeping_2.rb
@@ -0,0 +1,5 @@
+require './spec/cases/helper'
+
+Parallel.in_processes(5) do
+ sleep 2
+end
diff --git a/spec/cases/parallel_start_and_kill.rb b/spec/cases/parallel_start_and_kill.rb
new file mode 100644
index 0000000..3ab6c94
--- /dev/null
+++ b/spec/cases/parallel_start_and_kill.rb
@@ -0,0 +1,19 @@
+require './spec/cases/helper'
+
+method = case ARGV[0]
+when "PROCESS" then :in_processes
+when "THREAD" then :in_threads
+else raise "Learn to use this!"
+end
+
+options = {}
+options[:count] = 2
+if ARGV.length > 1
+ options[:interrupt_signal] = ARGV[1].to_s
+ trap('SIGINT') { puts 'Wrapper caught SIGINT' } if ARGV[1] != 'SIGINT'
+end
+
+Parallel.send(method, options) do
+ sleep 5
+ puts "I should have been killed earlier..."
+end
diff --git a/spec/cases/parallel_with_detected_cpus.rb b/spec/cases/parallel_with_detected_cpus.rb
new file mode 100644
index 0000000..471aacf
--- /dev/null
+++ b/spec/cases/parallel_with_detected_cpus.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+x = Parallel.in_processes do
+ "HELLO"
+end
+puts x
diff --git a/spec/cases/parallel_with_nil_uses_detected_cpus.rb b/spec/cases/parallel_with_nil_uses_detected_cpus.rb
new file mode 100644
index 0000000..2753f80
--- /dev/null
+++ b/spec/cases/parallel_with_nil_uses_detected_cpus.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+x = Parallel.in_processes(nil) do
+ "HELLO"
+end
+puts x
diff --git a/spec/cases/parallel_with_set_processes.rb b/spec/cases/parallel_with_set_processes.rb
new file mode 100644
index 0000000..e3cca0b
--- /dev/null
+++ b/spec/cases/parallel_with_set_processes.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+x = Parallel.in_processes(5) do
+ "HELLO"
+end
+puts x
diff --git a/spec/cases/profile_memroy.rb b/spec/cases/profile_memroy.rb
new file mode 100644
index 0000000..38ac501
--- /dev/null
+++ b/spec/cases/profile_memroy.rb
@@ -0,0 +1,23 @@
+def count_objects
+ old = Hash.new(0)
+ cur = Hash.new(0)
+ GC.start
+ ObjectSpace.each_object { |o| old[o.class] += 1 }
+ yield
+ GC.start
+ GC.start
+ ObjectSpace.each_object { |o| cur[o.class] += 1 }
+ Hash[cur.map{|k,v| [k, v - old[k]] }].reject{|k,v|v==0}
+end
+
+require './spec/cases/helper'
+
+items = Array.new(1000)
+options = {"in_#{ARGV[0]}".to_sym => 2}
+
+# TODO not sure why this fails without 2.times in threading mode :(
+
+puts(count_objects { 2.times { Parallel.map(items, options) {} } }.inspect)
+
+puts(count_objects { 2.times { Parallel.map(items, options) {} } }.inspect)
+
diff --git a/spec/cases/progress.rb b/spec/cases/progress.rb
new file mode 100644
index 0000000..8099895
--- /dev/null
+++ b/spec/cases/progress.rb
@@ -0,0 +1,6 @@
+require './spec/cases/helper'
+
+title = (ENV["TITLE"] == "true" ? true : "Doing stuff")
+Parallel.map(1..50, :progress => title) do
+ sleep 1 if $stdout.tty? # for debugging
+end
diff --git a/spec/cases/progress_with_finish.rb b/spec/cases/progress_with_finish.rb
new file mode 100644
index 0000000..a91bd67
--- /dev/null
+++ b/spec/cases/progress_with_finish.rb
@@ -0,0 +1,11 @@
+require './spec/cases/helper'
+
+sum = 0
+finish = lambda { |item, index, result| sum += result }
+
+Parallel.map(1..50, :progress => "Doing stuff", :finish => finish) do
+ sleep 1 if $stdout.tty? # for debugging
+ 2
+end
+
+puts sum
diff --git a/spec/cases/progress_with_options.rb b/spec/cases/progress_with_options.rb
new file mode 100644
index 0000000..e8cd59d
--- /dev/null
+++ b/spec/cases/progress_with_options.rb
@@ -0,0 +1,27 @@
+require './spec/cases/helper'
+
+# ruby-progressbar ignores the format string you give it
+# unless the output is a TTY. When running in the test,
+# the output is not a TTY, so we cannot test that the format
+# string you pass overrides parallel's default. So, we pretend
+# that stdout is a TTY to test that the options are merged
+# in the correct way.
+tty_stdout = $stdout
+class << tty_stdout
+ def tty?
+ true
+ end
+end
+
+parallel_options = {
+ :progress => {
+ :title => "Reticulating Splines",
+ :progress_mark => ';',
+ :format => "%t %w",
+ :output => tty_stdout
+ }
+}
+
+Parallel.map(1..50, parallel_options) do
+ 2
+end
diff --git a/spec/cases/synchronizes_start_and_finish.rb b/spec/cases/synchronizes_start_and_finish.rb
new file mode 100644
index 0000000..091793e
--- /dev/null
+++ b/spec/cases/synchronizes_start_and_finish.rb
@@ -0,0 +1,16 @@
+require './spec/cases/helper'
+
+start = lambda {|item,index|
+ print item * 5
+ sleep rand * 0.2
+ puts item * 5
+}
+finish = lambda {|item,index,result|
+ print result * 5
+ sleep rand * 0.2
+ puts result * 5
+}
+Parallel.map(['a', 'b', 'c'], :start => start, :finish => finish) do |i|
+ sleep rand * 0.2
+ i.upcase
+end
diff --git a/spec/cases/with_break.rb b/spec/cases/with_break.rb
new file mode 100644
index 0000000..210c047
--- /dev/null
+++ b/spec/cases/with_break.rb
@@ -0,0 +1,13 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+result = Parallel.public_send(method, 1..100, in_worker_type => 4) do |x|
+ sleep 0.1 # so all workers get started
+ print x
+ raise Parallel::Break if x == 1
+ sleep 0.2 # so now no work gets queued before Parallel::Break is raised
+ x
+end
+print " Parallel::Break raised - result #{result.inspect}"
diff --git a/spec/cases/with_break_before_finish.rb b/spec/cases/with_break_before_finish.rb
new file mode 100644
index 0000000..19b3875
--- /dev/null
+++ b/spec/cases/with_break_before_finish.rb
@@ -0,0 +1,19 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+finish = lambda do |_item, _index, _result|
+ sleep 0.1
+ print "finish hook called"
+end
+
+result = Parallel.public_send(method, 1..100, in_worker_type => 4, finish: finish) do |x|
+ sleep 0.1 # let workers start
+ raise Parallel::Break if x == 1
+ sleep 0.2
+ print x
+ x
+end
+
+print " Parallel::Break raised"
diff --git a/spec/cases/with_exception.rb b/spec/cases/with_exception.rb
new file mode 100644
index 0000000..39b6a64
--- /dev/null
+++ b/spec/cases/with_exception.rb
@@ -0,0 +1,16 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+begin
+ Parallel.public_send(method, 1..100, in_worker_type => 4) do |x|
+ sleep 0.1 # so all workers get started
+ print x
+ raise 'foo' if x == 1
+ sleep 0.2 # so now no work gets queued before exception is raised
+ x
+ end
+rescue
+ print ' raised'
+end
diff --git a/spec/cases/with_exception_before_finish.rb b/spec/cases/with_exception_before_finish.rb
new file mode 100644
index 0000000..b185d04
--- /dev/null
+++ b/spec/cases/with_exception_before_finish.rb
@@ -0,0 +1,23 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+class ParallelTestError < StandardError
+end
+
+begin
+ finish = lambda do |_item, _index, _result|
+ print " called"
+ end
+
+ Parallel.public_send(method, 1..10, in_worker_type => 4, finish: finish) do |x|
+ if x != 3
+ sleep 0.2
+ raise ParallelTestError
+ end
+ print x
+ x
+ end
+rescue ParallelTestError
+end
diff --git a/spec/cases/with_exception_in_finish.rb b/spec/cases/with_exception_in_finish.rb
new file mode 100644
index 0000000..11cfc44
--- /dev/null
+++ b/spec/cases/with_exception_in_finish.rb
@@ -0,0 +1,19 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+begin
+ finish = lambda do |x, _index, _result|
+ raise 'foo' if x == 1
+ end
+
+ Parallel.public_send(method, 1..100, in_worker_type => 4, finish: finish) do |x|
+ sleep 0.1 # so all workers get started
+ print x
+ sleep 0.2 unless x == 1 # so now no work gets queued before exception is raised
+ x
+ end
+rescue
+ print ' raised'
+end
diff --git a/spec/cases/with_exception_in_start.rb b/spec/cases/with_exception_in_start.rb
new file mode 100644
index 0000000..ed40e63
--- /dev/null
+++ b/spec/cases/with_exception_in_start.rb
@@ -0,0 +1,19 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+begin
+ start = lambda do |_item, _index|
+ @started = @started ? @started + 1 : 1
+ raise 'foo' if @started == 4
+ end
+
+ Parallel.public_send(method, 1..100, in_worker_type => 4, start: start) do |x|
+ print x
+ sleep 0.2 # so now no work gets queued before exception is raised
+ x
+ end
+rescue
+ print ' raised'
+end
diff --git a/spec/cases/with_exception_in_start_before_finish.rb b/spec/cases/with_exception_in_start_before_finish.rb
new file mode 100644
index 0000000..06ee056
--- /dev/null
+++ b/spec/cases/with_exception_in_start_before_finish.rb
@@ -0,0 +1,26 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+class ParallelTestError < StandardError
+end
+
+begin
+ start = lambda do |item, _index|
+ if item != 3
+ sleep 0.2
+ raise ParallelTestError
+ end
+ end
+
+ finish = lambda do |_item, _index, _result|
+ print " called"
+ end
+
+ Parallel.public_send(method, 1..10, in_worker_type => 4, start: start, finish: finish) do |x|
+ print x
+ x
+ end
+rescue ParallelTestError
+end
diff --git a/spec/cases/with_lambda.rb b/spec/cases/with_lambda.rb
new file mode 100644
index 0000000..4143299
--- /dev/null
+++ b/spec/cases/with_lambda.rb
@@ -0,0 +1,12 @@
+require './spec/cases/helper'
+
+type = case ARGV[0]
+when "PROCESSES" then :in_processes
+when "THREADS" then :in_threads
+else
+ raise "Use PROCESSES or THREADS"
+end
+
+all = [3,2,1]
+produce = lambda { all.pop || Parallel::Stop }
+puts Parallel.map(produce, type => 2) { |(i, id)| "ITEM-#{i}" }
diff --git a/spec/cases/with_queue.rb b/spec/cases/with_queue.rb
new file mode 100644
index 0000000..307808d
--- /dev/null
+++ b/spec/cases/with_queue.rb
@@ -0,0 +1,18 @@
+require './spec/cases/helper'
+
+type = case ARGV[0]
+when "PROCESSES" then :in_processes
+when "THREADS" then :in_threads
+else
+ raise "Use PROCESSES or THREADS"
+end
+
+queue = Queue.new
+Thread.new do
+ sleep 0.2
+ queue.push 1
+ queue.push 2
+ queue.push 3
+ queue.push Parallel::Stop
+end
+puts Parallel.map(queue, type => 2) { |(i, id)| "ITEM-#{i}" }
diff --git a/spec/cases/with_worker_number.rb b/spec/cases/with_worker_number.rb
new file mode 100644
index 0000000..5b49695
--- /dev/null
+++ b/spec/cases/with_worker_number.rb
@@ -0,0 +1,10 @@
+require './spec/cases/helper'
+
+method = ENV.fetch('METHOD')
+in_worker_type = "in_#{ENV.fetch('WORKER_TYPE')}".to_sym
+
+Parallel.public_send(method, 1..100, in_worker_type => 4) do
+ sleep 0.1 # so all workers get started
+ print Parallel.worker_number
+end
+
diff --git a/spec/parallel_spec.rb b/spec/parallel_spec.rb
new file mode 100644
index 0000000..0572b23
--- /dev/null
+++ b/spec/parallel_spec.rb
@@ -0,0 +1,554 @@
+require 'spec_helper'
+
+describe Parallel do
+ worker_types = (Process.respond_to?(:fork) ? ["processes", "threads"] : ["threads"])
+
+ def time_taken
+ t = Time.now.to_f
+ yield
+ RUBY_ENGINE == "jruby" ? 0: Time.now.to_f - t # jruby is super slow ... don't blow up all the tests ...
+ end
+
+ def kill_process_with_name(file, signal='INT')
+ running_processes = `ps -f`.split("\n").map{ |line| line.split(/\s+/) }
+ pid_index = running_processes.detect { |p| p.include?("UID") }.index("UID") + 1
+ parent_pid = running_processes.detect { |p| p.include?(file) and not p.include?("sh") }[pid_index]
+ `kill -s #{signal} #{parent_pid}`
+ end
+
+ def execute_start_and_kill(command, amount, signal='INT')
+ t = nil
+ lambda {
+ t = Thread.new { `ruby spec/cases/parallel_start_and_kill.rb #{command} 2>&1 && echo "FINISHED"` }
+ sleep 1.5
+ kill_process_with_name('spec/cases/parallel_start_and_kill.rb', signal)
+ sleep 1
+ }.should change { `ps`.split("\n").size }.by amount
+ t.value
+ end
+
+ describe ".processor_count" do
+ before do
+ Parallel.instance_variable_set(:@processor_count, nil)
+ end
+
+ it "returns a number" do
+ (1..999).should include(Parallel.processor_count)
+ end
+
+ if RUBY_PLATFORM =~ /darwin10/
+ it 'works if hwprefs in not available' do
+ Parallel.should_receive(:hwprefs_available?).and_return false
+ (1..999).should include(Parallel.processor_count)
+ end
+ end
+ end
+
+ describe ".physical_processor_count" do
+ before do
+ Parallel.instance_variable_set(:@physical_processor_count, nil)
+ end
+
+ it "returns a number" do
+ (1..999).should include(Parallel.physical_processor_count)
+ end
+
+ it "is even factor of logical cpus" do
+ (Parallel.processor_count % Parallel.physical_processor_count).should == 0
+ end
+ end
+
+ describe ".in_processes" do
+ def cpus
+ Parallel.processor_count
+ end
+
+ it "executes with detected cpus" do
+ `ruby spec/cases/parallel_with_detected_cpus.rb`.should == "HELLO\n" * cpus
+ end
+
+ it "executes with detected cpus when nil was given" do
+ `ruby spec/cases/parallel_with_nil_uses_detected_cpus.rb`.should == "HELLO\n" * cpus
+ end
+
+ it "set amount of parallel processes" do
+ `ruby spec/cases/parallel_with_set_processes.rb`.should == "HELLO\n" * 5
+ end
+
+ it "does not influence outside data" do
+ `ruby spec/cases/parallel_influence_outside_data.rb`.should == "yes"
+ end
+
+ it "kills the processes when the main process gets killed through ctrl+c" do
+ time_taken {
+ result = execute_start_and_kill "PROCESS", 0
+ result.should_not include "FINISHED"
+ }.should be <= 3
+ end
+
+ it "kills the processes when the main process gets killed through a custom interrupt" do
+ time_taken {
+ execute_start_and_kill "PROCESS SIGTERM", 0, "TERM"
+ }.should be <= 3
+ end
+
+ it "kills the threads when the main process gets killed through ctrl+c" do
+ time_taken {
+ result = execute_start_and_kill "THREAD", 0
+ result.should_not include "FINISHED"
+ }.should be <= 3
+ end
+
+ it "does not kill processes when the main process gets sent an interrupt besides the custom interrupt" do
+ time_taken {
+ result = execute_start_and_kill "PROCESS SIGTERM", 4
+ result.should include 'FINISHED'
+ result.should include 'Wrapper caught SIGINT'
+ result.should include 'I should have been killed earlier'
+ }.should be <= 7
+ end
+
+ it "does not kill threads when the main process gets sent an interrupt besides the custom interrupt" do
+ time_taken {
+ result = execute_start_and_kill "THREAD SIGTERM", 2
+ result.should include 'FINISHED'
+ result.should include 'Wrapper caught SIGINT'
+ result.should include 'I should have been killed earlier'
+ }.should be <= 7
+ end
+
+ it "does not kill anything on ctrl+c when everything has finished" do
+ time_taken do
+ t = Thread.new { `ruby spec/cases/parallel_fast_exit.rb 2>&1` }
+ sleep 2
+ kill_process_with_name("spec/cases/parallel_fast_exit.rb") #simulates Ctrl+c
+ sleep 1
+ result = t.value
+ result.scan(/I finished/).size.should == 3
+ result.should_not include("Parallel execution interrupted")
+ end.should <= 4
+ end
+
+ it "preserves original intrrupts" do
+ t = Thread.new { `ruby spec/cases/double_interrupt.rb 2>&1 && echo FIN` }
+ sleep 2
+ kill_process_with_name("spec/cases/double_interrupt.rb") #simulates Ctrl+c
+ sleep 1
+ result = t.value
+ result.should include("YES")
+ result.should include("FIN")
+ end
+
+ it "restores original intrrupts" do
+ `ruby spec/cases/after_interrupt.rb 2>&1`.should == "DEFAULT\n"
+ end
+
+ it "saves time" do
+ time_taken{
+ `ruby spec/cases/parallel_sleeping_2.rb`
+ }.should < 3.5
+ end
+
+ it "raises when one of the processes raises" do
+ `ruby spec/cases/parallel_raise.rb`.strip.should == 'TEST'
+ end
+
+ it "can raise an undumpable exception" do
+ `ruby spec/cases/parallel_raise_undumpable.rb`.strip.should include('Undumpable Exception')
+ end
+
+ it 'can handle to high fork rate' do
+ unless RbConfig::CONFIG["target_os"] =~ /darwin1/
+ `ruby spec/cases/parallel_high_fork_rate.rb`.should == 'OK'
+ end
+ end
+
+ it 'does not leave processes behind while running' do
+ skip if ENV['TRAVIS'] # this randomly fails on travis all the time :(
+ `ruby spec/cases/closes_processes_at_runtime.rb`.should == 'OK'
+ end
+
+ it "does not open unnecessary pipes" do
+ open_pipes = `lsof | grep pipe | wc -l`.to_i
+ max_pipes = `ruby spec/cases/count_open_pipes.rb`.to_i
+ (max_pipes - open_pipes).should < 400
+ end
+ end
+
+ describe ".in_threads" do
+ it "saves time" do
+ time_taken{
+ Parallel.in_threads(3){ sleep 2 }
+ }.should < 3
+ end
+
+ it "does not create new processes" do
+ lambda{ Thread.new{ Parallel.in_threads(2){sleep 1} } }.should_not change{`ps`.split("\n").size}
+ end
+
+ it "returns results as array" do
+ Parallel.in_threads(4){|i| "XXX#{i}"}.should == ["XXX0",'XXX1','XXX2','XXX3']
+ end
+
+ it "raises when a thread raises" do
+ lambda{ Parallel.in_threads(2){|i| raise "TEST"} }.should raise_error("TEST")
+ end
+ end
+
+ describe ".map" do
+ it "saves time" do
+ time_taken{
+ `ruby spec/cases/parallel_map_sleeping.rb`
+ }.should <= 3.5
+ end
+
+ it "executes with given parameters" do
+ `ruby spec/cases/parallel_map.rb`.should == "-a- -b- -c- -d-"
+ end
+
+ it "can dump/load complex objects" do
+ `ruby spec/cases/parallel_map_complex_objects.rb`.should == "YES"
+ end
+
+ it "starts new process imediatly when old exists" do
+ time_taken{
+ `ruby spec/cases/parallel_map_uneven.rb`
+ }.should <= 3.5
+ end
+
+ it "does not flatten results" do
+ Parallel.map([1,2,3], :in_threads=>2){|x| [x,x]}.should == [[1,1],[2,2],[3,3]]
+ end
+
+ it "can run in threads" do
+ Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads=>4){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
+ end
+
+ it 'supports all Enumerable-s' do
+ `ruby spec/cases/parallel_map_range.rb`.should == '[1, 2, 3, 4, 5]'
+ end
+
+ it 'handles nested arrays and nil correctly' do
+ `ruby spec/cases/map_with_nested_arrays_and_nil.rb`.should == '[nil, [2, 2], [[3], [3]]]'
+ end
+
+ worker_types.each do |type|
+ it "stops all workers when one fails in #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception.rb 2>&1`.should =~ /^\d{4} raised$/
+ end
+
+ it "stops all workers when one raises Break in #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break.rb 2>&1`.should =~ /^\d{4} Parallel::Break raised - result nil$/
+ end
+
+ it "stops all workers when a start hook fails with #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start.rb 2>&1`.should =~ /^\d{3} raised$/
+ end
+
+ it "stops all workers when a finish hook fails with #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_finish.rb 2>&1`.should =~ /^\d{4} raised$/
+ end
+
+ it "does not call the finish hook when a worker fails with #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_before_finish.rb 2>&1`.should == '3 called'
+ end
+
+ it "does not call the finish hook when a worker raises Break in #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_break_before_finish.rb 2>&1`.should =~ /^\d{3}(finish hook called){3} Parallel::Break raised$/
+ end
+
+ it "does not call the finish hook when a start hook fails with #{type}" do
+ `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1`.should == '3 called'
+ end
+
+ it "sets Parallel.worker_number with 4 #{type}" do
+ out = `METHOD=map WORKER_TYPE=#{type} ruby spec/cases/with_worker_number.rb 2>&1`
+ out.should =~ /\A[0123]+\z/
+ %w(0 1 2 3).each { |number| out.should include number }
+ end
+
+ it "sets Parallel.worker_number with 0 #{type}" do
+ type_key = "in_#{type}".to_sym
+ Parallel.map([1,2,3,4,5,6,7,8,9], type_key => 0) { |x| Parallel.worker_number }.uniq.should == [0]
+ Parallel.worker_number.should be_nil
+ end
+ end
+
+ it "can run with 0 threads" do
+ Thread.should_not_receive(:exclusive)
+ Parallel.map([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
+ end
+
+ it "can run with 0 processes" do
+ Process.should_not_receive(:fork)
+ Parallel.map([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x| x+2 }.should == [3,4,5,6,7,8,9,10,11]
+ end
+
+ it "notifies when an item of work is dispatched to a worker process" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0)
+ monitor.should_receive(:call).once.with(:second, 1)
+ monitor.should_receive(:call).once.with(:third, 2)
+ Parallel.map([:first, :second, :third], :start => monitor, :in_processes => 3) {}
+ end
+
+ it "notifies when an item of work is dispatched with 0 processes" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0)
+ monitor.should_receive(:call).once.with(:second, 1)
+ monitor.should_receive(:call).once.with(:third, 2)
+ Parallel.map([:first, :second, :third], :start => monitor, :in_processes => 0) {}
+ end
+
+ it "notifies when an item of work is completed by a worker process" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0, 123)
+ monitor.should_receive(:call).once.with(:second, 1, 123)
+ monitor.should_receive(:call).once.with(:third, 2, 123)
+ Parallel.map([:first, :second, :third], :finish => monitor, :in_processes => 3) { 123 }
+ end
+
+ it "notifies when an item of work is completed with 0 processes" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0, 123)
+ monitor.should_receive(:call).once.with(:second, 1, 123)
+ monitor.should_receive(:call).once.with(:third, 2, 123)
+ Parallel.map([:first, :second, :third], :finish => monitor, :in_processes => 0) { 123 }
+ end
+
+ it "notifies when an item of work is dispatched to a threaded worker" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0)
+ monitor.should_receive(:call).once.with(:second, 1)
+ monitor.should_receive(:call).once.with(:third, 2)
+ Parallel.map([:first, :second, :third], :start => monitor, :in_threads => 3) {}
+ end
+
+ it "notifies when an item of work is dispatched with 0 threads" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0)
+ monitor.should_receive(:call).once.with(:second, 1)
+ monitor.should_receive(:call).once.with(:third, 2)
+ Parallel.map([:first, :second, :third], :start => monitor, :in_threads => 0) {}
+ end
+
+ it "notifies when an item of work is completed by a threaded worker" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0, 123)
+ monitor.should_receive(:call).once.with(:second, 1, 123)
+ monitor.should_receive(:call).once.with(:third, 2, 123)
+ Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 3) { 123 }
+ end
+
+ it "notifies when an item of work is completed with 0 threads" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0, 123)
+ monitor.should_receive(:call).once.with(:second, 1, 123)
+ monitor.should_receive(:call).once.with(:third, 2, 123)
+ Parallel.map([:first, :second, :third], :finish => monitor, :in_threads => 0) { 123 }
+ end
+
+ it "spits out a useful error when a worker dies before read" do
+ `ruby spec/cases/map_with_killed_worker_before_read.rb 2>&1`.should include "DEAD"
+ end
+
+ it "spits out a useful error when a worker dies before write" do
+ `ruby spec/cases/map_with_killed_worker_before_write.rb 2>&1`.should include "DEAD"
+ end
+
+ it "raises DeadWorker when using exit so people learn to not kill workers and do not crash main process" do
+ `ruby spec/cases/exit_in_process.rb 2>&1`.should include "Yep, DEAD"
+ end
+
+ it 'raises EOF (not DeadWorker) when a worker raises EOF in process' do
+ `ruby spec/cases/eof_in_process.rb 2>&1`.should include 'Yep, EOF'
+ end
+
+ it "can be killed instantly" do
+ result = `ruby spec/cases/parallel_kill.rb 2>&1`
+ result.should == "DEAD\nWorks nil\n"
+ end
+
+ it "synchronizes :start and :finish" do
+ out = `ruby spec/cases/synchronizes_start_and_finish.rb`
+ %w{a b c}.each {|letter|
+ out.sub! letter.downcase * 10, 'OK'
+ out.sub! letter.upcase * 10, 'OK'
+ }
+ out.should == "OK\n" * 6
+ end
+
+ it 'is equivalent to serial map' do
+ l = Array.new(10_000){|i| i}
+ Parallel.map(l, {in_threads: 4}){|x| x+1}.should == l.map{|x| x+1}
+ end
+
+ it 'can work in isolation' do
+ out = `ruby spec/cases/map_isolation.rb`
+ out.should == "1\n2\n3\n4\nOK"
+ end
+
+ it 'sets Parallel.worker_number when run with isolation' do
+ out = `ruby spec/cases/map_worker_number_isolation.rb`
+ out.should == "0,1\nOK"
+ end
+
+ end
+
+ describe ".map_with_index" do
+ it "yields object and index" do
+ `ruby spec/cases/map_with_index.rb 2>&1`.should == 'a0b1'
+ end
+
+ it "does not crash with empty set" do
+ `ruby spec/cases/map_with_index_empty.rb 2>&1`.should == ''
+ end
+
+ it "can run with 0 threads" do
+ Thread.should_not_receive(:exclusive)
+ Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_threads => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11]
+ end
+
+ it "can run with 0 processes" do
+ Process.should_not_receive(:fork)
+ Parallel.map_with_index([1,2,3,4,5,6,7,8,9], :in_processes => 0){|x,i| x+2 }.should == [3,4,5,6,7,8,9,10,11]
+ end
+ end
+
+ describe ".each" do
+ it "returns original array, works like map" do
+ `ruby spec/cases/each.rb`.should == 'a b c d'
+ end
+
+ it "passes result to :finish callback :in_processes`" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0, 123)
+ monitor.should_receive(:call).once.with(:second, 1, 123)
+ monitor.should_receive(:call).once.with(:third, 2, 123)
+ Parallel.each([:first, :second, :third], :finish => monitor, :in_processes => 3) { 123 }
+ end
+
+ it "passes result to :finish callback :in_threads`" do
+ monitor = double('monitor', :call => nil)
+ monitor.should_receive(:call).once.with(:first, 0, 123)
+ monitor.should_receive(:call).once.with(:second, 1, 123)
+ monitor.should_receive(:call).once.with(:third, 2, 123)
+ Parallel.each([:first, :second, :third], :finish => monitor, :in_threads => 3) { 123 }
+ end
+
+ it "does not use marshal_dump" do
+ `ruby spec/cases/no_dump_with_each.rb 2>&1`.should == 'no dump for resultno dump for each'
+ end
+
+ it "does not slow down with lots of GC work in threads" do
+ Benchmark.realtime { `ruby spec/cases/no_gc_with_each.rb 2>&1` }.should <= (ENV["TRAVIS"] ? 15 : 10)
+ end
+
+ it "can modify in-place" do
+ `ruby spec/cases/each_in_place.rb`.should == 'ab'
+ end
+
+ worker_types.each do |type|
+ it "works with SQLite in #{type}" do
+ `WORKER_TYPE=#{type} ruby spec/cases/each_with_ar_sqlite.rb 2>&1`.should == "Parent: X\nParallel (in_#{type}): XXX\nParent: X\n"
+ end
+
+ it "stops all workers when one fails in #{type}" do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception.rb 2>&1`.should =~ /^\d{4} raised$/
+ end
+
+ it "stops all workers when one raises Break in #{type}" do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_break.rb 2>&1`.should =~ /^\d{4} Parallel::Break raised - result 1\.\.100$/
+ end
+
+ it "stops all workers when a start hook fails with #{type}" do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start.rb 2>&1`.should =~ /^\d{3} raised$/
+ end
+
+ it 'stops all workers when a finish hook fails with processes' do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_finish.rb 2>&1`.should =~ /^\d{4} raised$/
+ end
+
+ it "does not call the finish hook when a worker fails with #{type}" do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_before_finish.rb 2>&1`.should == '3 called'
+ end
+
+ it "does not call the finish hook when a worker raises Break in #{type}" do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_break_before_finish.rb 2>&1`.should =~ /^\d{3}(finish hook called){3} Parallel::Break raised$/
+ end
+
+ it "does not call the finish hook when a start hook fails with #{type}" do
+ `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_exception_in_start_before_finish.rb 2>&1`.should == '3 called'
+ end
+
+ it "sets Parallel.worker_number with #{type}" do
+ out = `METHOD=each WORKER_TYPE=#{type} ruby spec/cases/with_worker_number.rb 2>&1`
+ out.should =~ /\A[0123]+\z/
+ %w(0 1 2 3).each { |number| out.should include number }
+ end
+ end
+ end
+
+ describe ".each_with_index" do
+ it "yields object and index" do
+ ["a0b1", "b1a0"].should include `ruby spec/cases/each_with_index.rb 2>&1`
+ end
+ end
+
+ describe "progress" do
+ it "takes the title from :progress" do
+ `ruby spec/cases/progress.rb 2>&1`.sub(/=+/, '==').strip.should == "Doing stuff: |==|"
+ end
+
+ it "takes true from :progress" do
+ `TITLE=true ruby spec/cases/progress.rb 2>&1`.sub(/=+/, '==').strip.should == "Progress: |==|"
+ end
+
+ it "works with :finish" do
+ `ruby spec/cases/progress_with_finish.rb 2>&1`.strip.sub(/=+/, '==').gsub(/\n+/,"\n").should == "Doing stuff: |==|\n100"
+ end
+
+ it "takes the title from :progress[:title] and passes options along" do
+ `ruby spec/cases/progress_with_options.rb 2>&1`.should =~ /Reticulating Splines ;+ \d+ ;+/
+ end
+ end
+
+ ["lambda", "queue"].each do |thing|
+ describe "lambdas" do
+ let(:result) { "ITEM-1\nITEM-2\nITEM-3\n" }
+
+ it "runs in threads" do
+ `ruby spec/cases/with_#{thing}.rb THREADS 2>&1`.should == result
+ end
+
+ it "runs in processs" do
+ `ruby spec/cases/with_#{thing}.rb PROCESSES 2>&1`.should == result
+ end
+
+ it "refuses to use progress" do
+ lambda {
+ Parallel.map(lambda{}, :progress => "xxx"){ raise "Ooops" }
+ }.should raise_error("Progressbar can only be used with array like items")
+ end
+ end
+ end
+
+ it "fails when running with a prefilled queue without stop since there are no threads to fill it" do
+ error = (RUBY_VERSION >= "2.0.0" ? "No live threads left. Deadlock?" : "deadlock detected (fatal)")
+ `ruby spec/cases/fatal_queue.rb 2>&1`.should include error
+ end
+
+ describe "GC" do
+ def normalize(result)
+ result.sub(/\{(.*)\}/, "\\1").split(", ").reject { |x| x =~ /^(Hash|Array|String)=>(1|-1|-2)$/ }
+ end
+
+ worker_types.each do |type|
+ it "does not leak memory in #{type}" do
+ pending if RUBY_ENGINE == 'jruby' # lots of objects ... GC does not seem to work ...
+ result = `ruby #{"-X+O" if RUBY_ENGINE == 'jruby'} spec/cases/profile_memroy.rb #{type} 2>&1`.strip.split("\n").last
+ normalize(result).should == []
+ end
+ end
+ end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
new file mode 100644
index 0000000..912324e
--- /dev/null
+++ b/spec/spec_helper.rb
@@ -0,0 +1,7 @@
+require 'parallel'
+require 'benchmark'
+
+RSpec.configure do |config|
+ config.expect_with(:rspec) { |c| c.syntax = :should }
+ config.mock_with(:rspec) { |c| c.syntax = :should }
+end
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-ruby-extras/ruby-parallel.git
More information about the Pkg-ruby-extras-commits
mailing list