Pubblicazioni

@inproceedings{amigo-etal-2020-effectiveness,

title = {An Effectiveness Metric for Ordinal Classification: Formal Properties and Experimental Results},

author = {Enrique Amigó and Julio Gonzalo and Stefano Mizzaro and Jorge Carrillo-de-Albornoz},

editor = {Dan Jurafsky and Joyce Chai and Natalie Schluter and Joel Tetreault},

url = {https://aclanthology.org/2020.acl-main.363},

doi = {10.18653/v1/2020.acl-main.363},

year  = {2020},

date = {2020-01-01},

booktitle = {Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},

pages = {3938–3949},

publisher = {Association for Computational Linguistics},

address = {Online},

abstract = {In Ordinal Classification tasks, items have to be assigned to classes that have a relative ordering, such as ``positive'', ``neutral'', ``negative'' in sentiment analysis. Remarkably, the most popular evaluation metrics for ordinal classification tasks either ignore relevant information (for instance, precision/recall on each of the classes ignores their relative ordering) or assume additional information (for instance, Mean Average Error assumes absolute distances between classes). In this paper we propose a new metric for Ordinal Classification, Closeness Evaluation Measure, that is rooted on Measurement Theory and Information Theory. Our theoretical analysis and experimental results over both synthetic data and data from NLP shared tasks indicate that the proposed metric captures quality aspects from different traditional tasks simultaneously. In addition, it generalizes some popular classification (nominal scale) and error minimization (interval scale) metrics, depending on the measurement scale in which it is instantiated.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{bias-fairness-19,

title = {Bias and Fairness in Effectiveness Evaluation by Means of Network Analysis and Mixture Models},

author = {Kevin Roitero and Michael Soprano and Stefano Mizzaro},

year  = {2019},

date = {2019-10-14},

booktitle = {CEUR Workshop Proceedings},

volume = {2441},

pages = {2},

publisher = {CEUR-WS},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/iir/RoiteroMS19,

title = {Bias and Fairness in Effectiveness Evaluation by Means of Network 

 Analysis and Mixture Models},

author = {Kevin Roitero and Stefano Mizzaro and Michael Soprano},

url = {http://ceur-ws.org/Vol-2441/paper4.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 10th Italian Information Retrieval Workshop, Padova, 

 Italy, September 16-18, 2019.},

pages = {6--7},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/sigir/SopranoRM19,

title = {HITS Hits Readersourcing: Validating Peer Review Alternatives Using 

 Network Analysis},

author = {Michael Soprano and Kevin Roitero and Stefano Mizzaro},

url = {http://ceur-ws.org/Vol-2414/paper7.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 4th Joint Workshop on Bibliometric-enhanced Information 

 Retrieval and Natural Language Processing for Digital Libraries (BIRNDL 2019) co-located with the 42nd International ACM SIGIR Conference 

 on Research and Development in Information Retrieval (SIGIR 2019), 

 Paris, France, July 25, 2019.},

pages = {70--82},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/sigir/ZampieriRCKM19,

title = {On Topic Difficulty in IR Evaluation: The Effect of Systems, Corpora, 

 and System Components},

author = {Fabio Zampieri and Kevin Roitero and Shane J Culpepper and Oren Kurland and Stefano Mizzaro},

url = {https://doi.org/10.1145/3331184.3331279},

doi = {10.1145/3331184.3331279},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 42nd International ACM SIGIR Conference on 

 Research and Development in Information Retrieval, SIGIR 2019, Paris, 

 France, July 21-25, 2019.},

pages = {909--912},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/wsdm/HanRGSCMD19,

title = {All Those Wasted Hours: On Task Abandonment in Crowdsourcing},

author = {Lei Han and Kevin Roitero and Ujwal Gadiraju and Cristina Sarasua and Alessandro Checco and Eddy Maddalena and Gianluca Demartini},

url = {https://doi.org/10.1145/3289600.3291035},

doi = {10.1145/3289600.3291035},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the Twelfth ACM International Conference on Web Search 

 and Data Mining, WSDM 2019, Melbourne, VIC, Australia, February 

 11-15, 2019},

pages = {321--329},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1007/978-3-030-11226-4_21,

title = {Crowdsourcing Peer Review: As We May Do},

author = {Michael Soprano and Stefano Mizzaro},

editor = {Paolo Manghi and Leonardo Candela and Gianmaria Silvello},

isbn = {978-3-030-11226-4},

year  = {2019},

date = {2019-01-01},

booktitle = {Digital Libraries: Supporting Open Science},

pages = {259--273},

publisher = {Springer International Publishing},

address = {Cham},

abstract = {This paper describes Readersourcing 2.0, an ecosystem providing an implementation of the Readersourcing approach proposed by Mizzaro [10]. Readersourcing is proposed as an alternative to the standard peer review activity that aims to exploit the otherwise lost opinions of readers. Readersourcing 2.0 implements two different models based on the so-called codetermination algorithms. We describe the requirements, present the overall architecture, and show how the end-user can interact with the system. Readersourcing 2.0 will be used in the future to study also other topics, like the idea of shepherding the users to achieve a better quality of the reviews and the differences between a review activity carried out with a single-blind or a double-blind approach.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3357384.3358123,

title = {Towards Stochastic Simulations of Relevance Profiles},

author = {Kevin Roitero and Andrea Brunello and Julián Urbano and Stefano Mizzaro},

url = {https://doi.org/10.1145/3357384.3358123},

doi = {10.1145/3357384.3358123},

isbn = {9781450369763},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 28th ACM International Conference on Information and Knowledge Management},

pages = {2217–2220},

publisher = {Association for Computing Machinery},

address = {Beijing, China},

series = {CIKM '19},

abstract = {Recently proposed methods allow the generation of simulated scores representing the values of an effectiveness metric, but they do not investigate the generation of the actual lists of retrieved documents. In this paper we address this limitation: we present an approach that exploits an evolutionary algorithm and, given a metric score, creates a simulated relevance profile (i.e., a ranked list of relevance values) that produces that score. We show how the simulated relevance profiles are realistic under various analyses.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3357384.3357988,

title = {On Transforming Relevance Scales},

author = {Lei Han and Kevin Roitero and Eddy Maddalena and Stefano Mizzaro and Gianluca Demartini},

url = {https://doi.org/10.1145/3357384.3357988},

doi = {10.1145/3357384.3357988},

isbn = {9781450369763},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 28th ACM International Conference on Information and Knowledge Management},

pages = {39–48},

publisher = {Association for Computing Machinery},

address = {Beijing, China},

series = {CIKM '19},

abstract = {Information Retrieval (IR) researchers have often used existing IR evaluation collections and transformed the relevance scale in which judgments have been collected, e.g., to use metrics that assume binary judgments like Mean Average Precision. Such scale transformations are often arbitrary (e.g., 0,1 mapped to 0 and 2,3 mapped to 1) and it is assumed that they have no impact on the results of IR evaluation. Moreover, the use of crowdsourcing to collect relevance judgments has become a standard methodology. When designing the crowdsourcing relevance judgment task, one of the decision to be made is the how granular the relevance scale used to collect judgments should be. Such decision has then repercussions on the metrics used to measure IR system effectiveness. In this paper we look at the effect of scale transformations in a systematic way. We perform extensive experiments to study the transformation of judgments from fine-grained to coarse-grained. We use different relevance judgments expressed on different relevance scales and either expressed by expert annotators or collected by means of crowdsourcing. The objective is to understand the impact of relevance scale transformations on IR evaluation outcomes and to draw conclusions on how to best transform judgments into a different scale, when necessary.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{workshop-paper-iir2019,

title = {Bias and Fairness in Effectiveness Evaluation by Means of Network Analysis and Mixture Models},

author = {Kevin Roitero Soprano},

editor = {Maristella Agosti and Emanuele Di Buccio and Massimo Melucci and Stefano Mizzaro and Gabriella Pasi and Fabrizio Silvestri},

url = {http://ceur-ws.org/Vol-2441/paper4.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 10th Italian Information Retrieval Workshop, Padova, Italy, September 16-18, 2019},

volume = {2441},

pages = {6--7},

publisher = {CEUR-WS.org},

series = {CEUR Workshop Proceedings},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{workshop-paper-birnld2019,

title = {HITS Hits Readersourcing: Validating Peer Review Alternatives Using Network Analysis},

author = {Michael Soprano and Kevin Roitero and Stefano Mizzaro},

editor = {Muthu Kumar Chandrasekaran and Philipp Mayr},

url = {http://ceur-ws.org/Vol-2414/paper7.pdf},

year  = {2019},

date = {2019-01-01},

booktitle = {Proceedings of the 4th Joint Workshop on Bibliometric-enhanced Information Retrieval and Natural Language Processing for Digital Libraries (BIRNDL 2019) co-located with the 42nd International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2019), Paris, France, July 25, 2019},

volume = {2414},

pages = {70--82},

publisher = {CEUR-WS.org},

series = {CEUR Workshop Proceedings},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{conference-paper-ircdl2019,

title = {Crowdsourcing Peer Review: As We May Do},

author = {Michael Soprano and Stefano Mizzaro},

editor = {Paolo Manghi and Leonardo Candela and Gianmaria Silvello},

url = {https://doi.org/10.1007/978-3-030-11226-4_21},

doi = {10.1007/978-3-030-11226-4_21},

year  = {2019},

date = {2019-01-01},

booktitle = {Digital Libraries: Supporting Open Science - 15th Italian Research Conference on Digital Libraries, IRCDL 2019, Pisa, Italy, January 31 - February 1, 2019, Proceedings},

volume = {988},

pages = {259--273},

publisher = {Springer},

series = {Communications in Computer and Information Science},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{conference-paper-aiudc2019,

title = {Crowdsourcing Peer Review in the Digital Humanities?},

author = {Michael Soprano and Stefano Mizzaro},

url = {http://aiucd2019.uniud.it/wp-content/uploads/2020/03/AIUCD2019-BoA_DEF.pdf},

isbn = {9788894253535},

year  = {2019},

date = {2019-01-01},

booktitle = {Book of Abstracts, 8th AIUCD Conference 2019 – Pedagogy, Teaching, and Research in the Age of Digital Humanities, page 251. Udine, Italy, 2019},

pages = {251},

address = {Udine, Italy},

series = {AIUCD '19},

abstract = {We propose an alternative approach to the standard peer review activity that aims to exploit the otherwise lost opinions of readers of publications which is called Readersourcing, originally proposed by Mizzaro [1]. Such an approach can be formalized by means of different models which share the same general principles. These models should be able to define a way, to measure the overall quality of a publication as well the reputation of a reader as an assessor; moreover, from these measures it should be possible to derive the reputation of a scholar as an author. We describe an ecosystem called Readersourcing 2.0 which provides an implementation for two Readersourcing models [2, 3] by outlining its goals and requirements. Readersourcing 2.0 will be used in the future to gather fresh data to analyze and validate.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{toapp:howmany,

title = {How Many Truth Levels? Six? One Hundred? Even more? Validating Truthfulness of Statements via Crowdsourcing},

author = {Kevin Roitero and Damiano Spina and Gianluca Demartini and Stefano Mizzaro},

year  = {2018},

date = {2018-01-01},

booktitle = {To appear in RDSM 2018 : Workshop on Rumours and Deception in Social Media.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210108,

title = {Effectiveness Evaluation with a Subset of Topics: A Practical Approach},

author = {Kevin Roitero and Michael Soprano and Stefano Mizzaro},

url = {https://doi.org/10.1145/3209978.3210108},

doi = {10.1145/3209978.3210108},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {1145–1148},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {Several researchers have proposed to reduce the number of topics used in TREC-like initiatives. One research direction that has been pursued is what is the optimal topic subset of a given cardinality that evaluates the systems/runs in the most accurate way. Such a research direction has been so far mainly theoretical, with almost no indication on how to select the few good topics in practice. We propose such a practical criterion for topic selection: we rely on the methods for automatic system evaluation without relevance judgments, and by running some experiments on several TREC collections we show that the topics selected on the basis of those evaluations are indeed more informative than random topics.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210084,

title = {IRevalOO: An Object Oriented Framework for Retrieval Evaluation},

author = {Kevin Roitero and Eddy Maddalena and Yannick Ponte and Stefano Mizzaro},

url = {https://doi.org/10.1145/3209978.3210084},

doi = {10.1145/3209978.3210084},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {913–916},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {We propose IRevalOO, a flexible Object Oriented framework that (i) can be used as-is as a replacement of the widely adopted trec_eval software, and (ii) can be easily extended (or "instantiated'', in framework terminology) to implement different scenarios of test collection based retrieval evaluation. Instances of IRevalOO can provide a usable and convenient alternative to the state-of-the-art software commonly used by different initiatives (TREC, NTCIR, CLEF, FIRE, etc.). Also, those instances can be easily adapted to satisfy future customization needs of researchers, as: implementing and experimenting with new metrics, even based on new notions of relevance; using different formats for system output and "qrels''; and in general visualizing, comparing, and managing retrieval evaluation results.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210229,

title = {CHEERS: CHeap & Engineered Evaluation of Retrieval Systems},

author = {Kevin Roitero},

url = {https://doi.org/10.1145/3209978.3210229},

doi = {10.1145/3209978.3210229},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {1467},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {In test collection based evaluation of retrieval effectiveness, many research investigated different directions for an economical and a semi-automatic evaluation of retrieval systems. Although several methods have been proposed and experimentally evaluated, their accuracy seems still limited. In this paper we present our proposal for a more engineered approach to information retrieval evaluation.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210052,

title = {On Fine-Grained Relevance Scales},

author = {Kevin Roitero and Eddy Maddalena and Gianluca Demartini and Stefano Mizzaro},

url = {https://doi.org/10.1145/3209978.3210052},

doi = {10.1145/3209978.3210052},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

urldate = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {675–684},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {In Information Retrieval evaluation, the classical approach of adopting binary relevance judgments has been replaced by multi-level relevance judgments and by gain-based metrics leveraging such multi-level judgment scales. Recent work has also proposed and evaluated unbounded relevance scales by means of Magnitude Estimation (ME) and compared them with multi-level scales. While ME brings advantages like the ability for assessors to always judge the next document as having higher or lower relevance than any of the documents they have judged so far, it also comes with some drawbacks. For example, it is not a natural approach for human assessors to judge items as they are used to do on the Web (e.g., 5-star rating). In this work, we propose and experimentally evaluate a bounded and fine-grained relevance scale having many of the advantages and dealing with some of the issues of ME. We collect relevance judgments over a 100-level relevance scale (S100) by means of a large-scale crowdsourcing experiment and compare the results with other relevance scales (binary, 4-level, and ME) showing the benefit of fine-grained scales over both coarse-grained and unbounded scales as well as highlighting some new results on ME. Our results show that S100 maintains the flexibility of unbounded scales like ME in providing assessors with ample choice when judging document relevance (i.e., assessors can fit relevance judgments in between of previously given judgments). It also allows assessors to judge on a more familiar scale (e.g., on 10 levels) and to perform efficiently since the very first judging task.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210146,

title = {Query Performance Prediction and Effectiveness Evaluation Without Relevance Judgments: Two Sides of the Same Coin},

author = {Stefano Mizzaro and Josiane Mothe and Kevin Roitero and Md Zia Ullah},

url = {https://doi.org/10.1145/3209978.3210146},

doi = {10.1145/3209978.3210146},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {1233–1236},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {Some methods have been developed for automatic effectiveness evaluation without relevance judgments. We propose to use those methods, and their combination based on a machine learning approach, for query performance prediction. Moreover, since predicting average precision as it is usually done in query performance prediction literature is sensitive to the reference system that is chosen, we focus on predicting the average of average precision values over several systems. Results of an extensive experimental evaluation on ten TREC collections show that our proposed methods outperform state-of-the-art query performance predictors.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3234944.3234958,

title = {A Formal Account of Effectiveness Evaluation and Ranking Fusion},

author = {Enrique Amigó and Fernando Giner and Stefano Mizzaro and Damiano Spina},

url = {https://doi.org/10.1145/3234944.3234958},

doi = {10.1145/3234944.3234958},

isbn = {9781450356565},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 2018 ACM SIGIR International Conference on Theory of Information Retrieval},

pages = {123–130},

publisher = {Association for Computing Machinery},

address = {Tianjin, China},

series = {ICTIR '18},

abstract = {This paper proposes a theoretical framework which models the information provided by retrieval systems in terms of Information Theory. The proposed framework allows to formalize: (i) system effectiveness as an information theoretic similarity between system outputs and human assessments, and (ii) ranking fusion as an information quantity measure. As a result, the proposed effectiveness metric improves popular metrics in terms of formal constraints. In addition, our empirical experiments suggest that it captures quality aspects from traditional metrics, while the reverse is not true. Our work also advances the understanding of theoretical foundations of the empirically known phenomenon of effectiveness increase when combining retrieval system outputs in an unsupervised manner.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210131b,

title = {Are We on the Right Track? An Examination of Information Retrieval Methodologies},

author = {Enrique Amigó and Hui Fang and Stefano Mizzaro and ChengXiang Zhai},

url = {https://doi.org/10.1145/3209978.3210131},

doi = {10.1145/3209978.3210131},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {997–1000},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {The unpredictability of user behavior and the need for effectiveness make it difficult to define a suitable research methodology for Information Retrieval (IR). In order to tackle this challenge, we categorize existing IR methodologies along two dimensions: (1) empirical vs. theoretical, and (2) top-down vs. bottom-up. The strengths and drawbacks of the resulting categories are characterized according to 6 desirable aspects. The analysis suggests that different methodologies are complementary and therefore, equally necessary. The categorization of the 167 full papers published in the last SIGIR (2016 and 2017) and ICTIR (2017) conferences suggest that most of existing work is empirical bottom-up, suggesting lack of some desirable aspects. With the hope of improving IR research practice, we propose a general methodology for IR that integrates the strengths of existing research methods.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{conference-paper-sigir2018,

title = {Effectiveness Evaluation with a Subset of Topics: A Practical Approach},

author = {Kevin Roitero and Michael Soprano and Stefano Mizzaro},

url = {https://doi.org/10.1145/3209978.3210108},

doi = {10.1145/3209978.3210108},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 41st International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2018). Ann Arbor Michigan, U.S.A, July 8-12, 2018. Conference Rank: GGS A++, Core A*},

pages = {1145–1148},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {Several researchers have proposed to reduce the number of topics used in TREC-like initiatives. One research direction that has been pursued is what is the optimal topic subset of a given cardinality that evaluates the systems/runs in the most accurate way. Such a research direction has been so far mainly theoretical, with almost no indication on how to select the few good topics in practice. We propose such a practical criterion for topic selection: we rely on the methods for automatic system evaluation without relevance judgments, and by running some experiments on several TREC collections we show that the topics selected on the basis of those evaluations are indeed more informative than random topics.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/cikm/MaddalenaCM18,

title = {Multidimensional News Quality: A Comparison of Crowdsourcing and 

 Nichesourcing},

author = {Eddy Maddalena and Davide Ceolin and Stefano Mizzaro},

editor = {Alfredo Cuzzocrea and Francesco Bonchi and Dimitrios Gunopulos},

url = {http://ceur-ws.org/Vol-2482/paper17.pdf},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the CIKM 2018 Workshops co-located with 27th ACM 

 International Conference on Information and Knowledge Management (CIKM 

 2018), Torino, Italy, October 22, 2018},

volume = {2482},

publisher = {CEUR-WS.org},

series = {CEUR Workshop Proceedings},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3209978.3210131,

title = {Are we on the Right Track? An Examination of Information Retrieval Methodologies},

author = {Enrique Amigó and Hui Fang and Stefano Mizzaro and ChengXiang Zhai},

url = {https://doi.org/10.1145/3209978.3210131},

doi = {10.1145/3209978.3210131},

isbn = {9781450356572},

year  = {2018},

date = {2018-01-01},

booktitle = {The 41st International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {997–1000},

publisher = {Association for Computing Machinery},

address = {Ann Arbor, MI, USA},

series = {SIGIR '18},

abstract = {The unpredictability of user behavior and the need for effectiveness make it difficult to define a suitable research methodology for Information Retrieval (IR). In order to tackle this challenge, we categorize existing IR methodologies along two dimensions: (1) empirical vs. theoretical, and (2) top-down vs. bottom-up. The strengths and drawbacks of the resulting categories are characterized according to 6 desirable aspects. The analysis suggests that different methodologies are complementary and therefore, equally necessary. The categorization of the 167 full papers published in the last SIGIR (2016 and 2017) and ICTIR (2017) conferences suggest that most of existing work is empirical bottom-up, suggesting lack of some desirable aspects. With the hope of improving IR research practice, we propose a general methodology for IR that integrates the strengths of existing research methods.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3234944.3234958b,

title = {A Formal Account of Effectiveness Evaluation and Ranking Fusion},

author = {Enrique Amigó and Fernando Giner and Stefano Mizzaro and Damiano Spina},

url = {https://doi.org/10.1145/3234944.3234958},

doi = {10.1145/3234944.3234958},

isbn = {9781450356565},

year  = {2018},

date = {2018-01-01},

booktitle = {Proceedings of the 2018 ACM SIGIR International Conference on Theory of Information Retrieval},

pages = {123–130},

publisher = {Association for Computing Machinery},

address = {Tianjin, China},

series = {ICTIR '18},

abstract = {This paper proposes a theoretical framework which models the information provided by retrieval systems in terms of Information Theory. The proposed framework allows to formalize: (i) system effectiveness as an information theoretic similarity between system outputs and human assessments, and (ii) ranking fusion as an information quantity measure. As a result, the proposed effectiveness metric improves popular metrics in terms of formal constraints. In addition, our empirical experiments suggest that it captures quality aspects from traditional metrics, while the reverse is not true. Our work also advances the understanding of theoretical foundations of the empirically known phenomenon of effectiveness increase when combining retrieval system outputs in an unsupervised manner.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/iir/RoiteroMS17,

title = {Economic Evaluation of Recommender Systems: A Proposal},

author = {Kevin Roitero and Stefano Mizzaro and Giuseppe Serra},

url = {http://ceur-ws.org/Vol-1911/8.pdf},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 8th Italian Information Retrieval Workshop, Lugano, 

 Switzerland, June 05-07, 2017.},

pages = {48--51},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3121050.3121060b,

title = {Considering Assessor Agreement in IR Evaluation},

author = {Eddy Maddalena and Kevin Roitero and Gianluca Demartini and Stefano Mizzaro},

url = {https://doi.org/10.1145/3121050.3121060},

doi = {10.1145/3121050.3121060},

isbn = {9781450344906},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the ACM SIGIR International Conference on Theory of Information Retrieval},

pages = {75–82},

publisher = {Association for Computing Machinery},

address = {New York, NY, USA},

abstract = {The agreement between relevance assessors is an important but understudied topic in the Information Retrieval literature because of the limited data available about documents assessed by multiple judges. This issue has gained even more importance recently in light of crowdsourced relevance judgments, where it is customary to gather many relevance labels for each topic-document pair. In a crowdsourcing setting, agreement is often even used as a proxy for quality, although without any systematic verification of the conjecture that higher agreement corresponds to higher quality. In this paper we address this issue and we study in particular: the effect of topic on assessor agreement; the relationship between assessor agreement and judgment quality; the effect of agreement on ranking systems according to their effectiveness; and the definition of an agreement-aware effectiveness metric that does not discard information about multiple judgments for the same document as it typically happens in a crowdsourcing setting.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{chifu:hal-01712541,

title = {Human-Based Query Difficulty Prediction},

author = {Adrian-Gabriel Chifu and Sébastien Déjean and Stefano Mizzaro and Josiane Mothe},

url = {https://hal.archives-ouvertes.fr/hal-01712541},

year  = {2017},

date = {2017-01-01},

booktitle = {39th European Colloquium on Information Retrieval (ECIR 2017)},

pages = {pp. 343-356},

address = {Aberdeen, Scotland, United Kingdom},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/3077136.3084369,

title = {Axiomatic Thinking for Information Retrieval: And Related Tasks},

author = {Enrique Amigó and Hui Fang and Stefano Mizzaro and ChengXiang Zhai},

url = {https://doi.org/10.1145/3077136.3084369},

doi = {10.1145/3077136.3084369},

isbn = {9781450350228},

year  = {2017},

date = {2017-01-01},

booktitle = {Proceedings of the 40th International ACM SIGIR Conference on Research and Development in Information Retrieval},

pages = {1419–1420},

publisher = {Association for Computing Machinery},

address = {Shinjuku, Tokyo, Japan},

series = {SIGIR '17},

abstract = {This is the first workshop on the emerging interdisciplinary research area of applying axiomatic thinking to information retrieval (IR) and related tasks. The workshop aims to help foster collaboration of researchers working on different perspectives of axiomatic thinking and encourage discussion and research on general methodological issues related to applying axiomatic thinking to IR and related tasks.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/iir/RoiteroM16,

title = {Improving the Efficiency of Retrieval Effectiveness Evaluation: Finding 

 a Few Good Topics with Clustering?},

author = {Kevin Roitero and Stefano Mizzaro},

url = {http://ceur-ws.org/Vol-1653/paper_4.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the 7th Italian Information Retrieval Workshop, Venezia, 

 Italy, May 30-31, 2016.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{DBLP:conf/ecir/PavanMBS16,

title = {Exploiting News to Categorize Tweets: Quantifying the Impact of Different 

 News Collections},

author = {Marco Pavan and Stefano Mizzaro and Matteo Bernardon and Ivan Scagnetto},

editor = {Miguel Martinez - and Udo Kruschwitz and Gabriella Kazai and Frank Hopfgartner and David Corney and Ricardo Campos and Dyaa Albakour},

url = {http://ceur-ws.org/Vol-1568/paper10.pdf},

year  = {2016},

date = {2016-01-01},

booktitle = {Proceedings of the First International Workshop on Recent Trends in 

 News Information Retrieval co-located with 38th European Conference 

 on Information Retrieval (ECIR 2016), Padua, Italy, March 20, 2016},

volume = {1568},

pages = {54--59},

publisher = {CEUR-WS.org},

series = {CEUR Workshop Proceedings},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{7264310b,

title = {Finding Important Locations: A Feature-Based Approach},

author = {M Pavan and S Mizzaro and I Scagnetto and A Beggiato},

doi = {10.1109/MDM.2015.11},

year  = {2015},

date = {2015-01-01},

booktitle = {2015 16th IEEE International Conference on Mobile Data Management},

volume = {1},

pages = {110-115},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1007/978-3-319-16354-3_56,

title = {Content-Based Similarity of Twitter Users},

author = {Stefano Mizzaro and Marco Pavan and Ivan Scagnetto},

editor = {Allan Hanbury and Gabriella Kazai and Andreas Rauber and Norbert Fuhr},

isbn = {978-3-319-16354-3},

year  = {2015},

date = {2015-01-01},

booktitle = {Advances in Information Retrieval},

pages = {507--512},

publisher = {Springer International Publishing},

address = {Cham},

abstract = {We propose a method for computing user similarity based on a network representing the semantic relationships between the words occurring in the same tweet and the related topics. We use such specially crafted network to define several user profiles to be compared with cosine similarity. We also describe an initial experimental activity to study the effectiveness on a limited dataset.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1007/978-3-319-16354-3_93,

title = {A Formal Approach to Effectiveness Metrics for Information Access: Retrieval, Filtering, and Clustering},

author = {Enrique Amigó and Julio Gonzalo and Stefano Mizzaro},

editor = {Allan Hanbury and Gabriella Kazai and Andreas Rauber and Norbert Fuhr},

isbn = {978-3-319-16354-3},

year  = {2015},

date = {2015-01-01},

booktitle = {Advances in Information Retrieval},

pages = {817–821},

publisher = {Springer International Publishing},

address = {Cham},

abstract = {In this tutorial we present a formal account of evaluation metrics for three of the most salient information related tasks: Retrieval, Clustering, and Filtering. We focus on the most popular metrics and, by exploiting measurement theory, we show some constraints for suitable metrics in each of the three tasks. We also systematically compare metrics according to how they satisfy such constraints, we provide criteria to select the most adequate metric for each specific information access task, and we discuss how to combine and weight metrics.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{10.1145/2600428.2602296,

title = {A general account of effectiveness metrics for information tasks: retrieval, filtering, and clustering},

author = {Enrique Amigó and Julio Gonzalo and Stefano Mizzaro},

url = {https://doi.org/10.1145/2600428.2602296},

doi = {10.1145/2600428.2602296},

isbn = {9781450322577},

year  = {2014},

date = {2014-01-01},

booktitle = {Proceedings of the 37th International ACM SIGIR Conference on Research & Development in Information Retrieval},

pages = {1289},

publisher = {Association for Computing Machinery},

address = {Gold Coast, Queensland, Australia},

series = {SIGIR '14},

abstract = {In this tutorial we will present, review, and compare the most popular evaluation metrics for some of the most salient information related tasks, covering: (i) Information Retrieval, (ii) Clustering, and (iii) Filtering. The tutorial will make a special emphasis on the specification of constraints for suitable metrics in each of the three tasks, and on the systematic comparison of metrics according to such constraints. The last part of the tutorial will investigate the challenge of combining and weighting metrics.},

keywords = {},

pubstate = {published},

tppubtype = {inproceedings}

}