@Misc{lemur,
  key =		 {Lemur Query Log Project},
  OPTauthor = 	 {},
  OPTtitle =	 {Lemur Query Log Project},
  OPThowpublished = {},
  OPTmonth = 	 {},
  year =	 {2009},
  note =	 {\url{http://lemurstudy.cs.umass.edu/}},
  OPTannote = 	 {}
}

@Misc{evaluatir,
  key =		 {EvaluatIR},
  OPTauthor = 	 {},
  title =	 {An Online Tool for Evaluating and Comparing IR Systems},
  OPThowpublished = {},
  OPTmonth = 	 {},
  year =	 {2009},
  note =	 {\url{http://evaluatir.org/}},
  OPTannote = 	 {}
}

@Article{spar:what06,
  author = 	 {Karen {Sparck~Jones}},
  title = 	 {What's the value of {TREC} -- is there a gap to jump
                  or a chasm to bridge?},
  journal = 	 {SIGIR Forum},
  year = 	 2006,
  volume =	 40,
  OPTnumber =	 1,
  pages =	 {10-20}, 
  note =         {\url{http://doi.acm.org/10.1145/1147197.1147198}},
}

@article{kell:eval09,
 author = {Kelly, Diane and Dumais, Susan and Pedersen, Jan O.},
 title = {Evaluation Challenges and Directions for Information-Seeking Support Systems},
 journal = {Computer},
 volume = {42},
 OPTnumber = {3},
 year = {2009},
 OPTissn = {0018-9162},
 pages = {60--66},
 note = {\url{http://dx.doi.org/10.1109/MC.2009.82}},
 OPTdoi = {http://dx.doi.org/10.1109/MC.2009.82},
 OPTpublisher = {IEEE Computer Society Press},
 OPTaddress = {Los Alamitos, CA, USA},
 }

@TechReport{cleverdon60,
  author =	 {C. W. Cleverdon},
  title =	 {Report on the First Stage of an
                  Investigation into the Comparative Efficiency of
                  Indexing Systems},
  institution =	 {College of Aeronautics, Cranfield UK},
  year =	 1960
}

@TechReport{cleverdon62,
  author =	 {C. W. Cleverdon},
  title =	 {Report on the Testing and Analysis of an
                  Investigation into the Comparative Efficiency of
                  Indexing Systems},
  institution =	 {College of Aeronautics, Cranfield UK},
  year =	 1962
}

@Article{cleverdon67,
  author =	 {C. W. Cleverdon},
  title =	 {The {C}ranfield tests on index language devices},
  journal =	 {Aslib},
  year =	 1967,
  volume =	 19,
  pages =	 {173--192}
}


@Article{call:meet07,
  author =	 {Jamie Callan and James Allan and Charles
                  L. A. Clarke and Susan Dumais and David A. Evans and
                  Mark Sanderson and ChengXiang Zhai},
  title =	 {Meeting of the {MINDS}: an information retrieval
                  research agenda},
  journal =	 {SIGIR Forum},
  year =	 2007,
  volume =	 41,
  OPTnumber =	 2,
  pages =	 {25--34},
  month =	 {December},
  note =		 {\url{http://doi.acm.org/10.1145/1328964.1328967}}
}


# Proceedings.

@InProceedings{1,
  author =       {Chris Buckley},
  title =        {Towards Good Evaluation of Individual Topics },
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {1},
  year =         2009}


@InProceedings{2,
  author =       {Susan Dumais},
  title =        {Evaluating {IR} in Situ },
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {2},
  year =         2009}


@InProceedings{3,
  author =       {Georges Dupret},
  title =        {User Models to Compare and Evaluate Web {IR} Metrics },
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {3},
  year =         2009}


@InProceedings{4,
  author =       {Stephen Robertson},
  title =        {Richer Theories, Richer Experiments },
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {4},
  year =         2009}


@InProceedings{5,
  author =       {David Hawking and Paul Thomas and Tom Gedeon and Tom Rowlands and Tim Jones},
  title =        {New methods for creating testfiles: Tuning enterprise search with {C-TEST}},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {5--6},
  year =         2009}


@InProceedings{7,
  author =       {Nicholas Belkin and Michael Cole and Jingjing Liu},
  title =        {A Model for Evaluation of Interactive Information Retrieval},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {7--8},
  year =         2009}


@InProceedings{9,
  author =       {Cecile Paris and Nathalie Colineau and Paul Thomas and Ross Wilkinson},
  title =        {Stakeholders and their respective costs-benefits in {IR} evaluation},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {9--10},
  year =         2009}


@InProceedings{11,
  author =       {Mark Smucker},
  title =        {A Plan for Making Information Retrieval Evaluation Synonymous with Human Performance Prediction},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {11--12},
  year =         2009}


@InProceedings{13,
  author =       {Sofia Stamou and Efthimis Efthimiadis},
  title =        {Queries without Clicks: Successful or Failed Searches?},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {13--14},
  year =         2009}


@InProceedings{15,
  author =       {Omar Alonso and Stefano Mizzaro},
  title =        {Can we get rid of {TREC} assessors? Using {M}echanical {T}urk for relevance assessment},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {15--16},
  year =         2009}


@InProceedings{17,
  author =       {Tom Crecelius and Ralf Schenkel},
  title =        {Evaluating Network-Aware Retrieval in Social Networks},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {17--18},
  year =         2009}


@InProceedings{19,
  author =       {Wei Che (Darren) Huang and Andrew Trotman and Shlomo Geva},
  title =        {A Virtual Evaluation Forum for Cross Language Link Discovery},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {19--20},
  year =         2009}


@InProceedings{21,
  author =       {Gabriella Kazai and Natasa Milic-Frayling},
  title =        {On the Evaluation of the Quality of Relevance Assessments Collected through Crowdsourcing},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {21--22},
  year =         2009}


@InProceedings{23,
  author =       {Zhen Yue and Abhay Harplale and Daqing He and Jonathan Grady and Yiling Lin and Jon Walker and Siddharth Gopal and Yiming Yang},
  title =        {{CiteEval} for Evaluating Personalized Social Web Search},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {23--24},
  year =         2009}


@InProceedings{25,
  author =       {Timothy Armstrong and Justin Zobel and William Webber and Alistair Moffat},
  title =        {Relative Significance is Insufficient: Baselines Matter Too},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {25--26},
  year =         2009}


@InProceedings{27,
  author =       {Kevyn Collins-Thompson},
  title =        {Accounting for stability of retrieval algorithms using risk-reward curves},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {27--28},
  year =         2009}


@InProceedings{29,
  author =       {Allan Hanbury and Henning M{\"u}ller},
  title =        {Toward Automated Component-Level Evaluation},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {29--30},
  year =         2009}


@InProceedings{31,
  author =       {Hua Liu and Ruihua Song and Jian-Yun Nie and Ji-Rong Wen},
  title =        {Building a Test Collection for Evaluating Search Result Diversity: A Preliminary Study},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {31--32},
  year =         2009}


@InProceedings{33,
  author =       {Milad Shokouhi and Emine Yilmaz and Nick Craswell and Stephen Robertson},
  title =        {Are Evaluation Metrics Identical With Binary Judgements?},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {33--34},
  year =         2009}


@InProceedings{35,
  author =       {Sadek Ali and Mariano Consens},
  title =        {Enhanced Web  Retrieval Task},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {35--36},
  year =         2009}


@InProceedings{37,
  author =       {Miguel Costa and M{\'a}rio Silva},
  title =        {Towards Information Retrieval Evaluation over Web Archives},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {37--38},
  year =         2009}


@InProceedings{39,
  author =       {Jinyoung Kim and Bruce Croft},
  title =        {Building Pseudo-Desktop Collections},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {39--40},
  year =         2009}


@InProceedings{41,
  author =       {Neal Lathia and Stephen Hailes and Licia Capra},
  title =        {Evaluating Collaborative Filtering Over Time},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {41--42},
  year =         2009}


@InProceedings{43,
  author =       {Fernando Llopis and Alberto Escapa and Antonio Ferrandez and Sergio Navarro and Elsia Noguera},
  title =        {How long can you wait for your {QA} system?},
  booktitle = {Proceedings of the SIGIR 2009 Workshop on the Future of IR Evaluation},
  pages =        {43--44},
  year =         2009}